2024-12-05 11:13:57,787 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 11:13:57,802 main DEBUG Took 0.012261 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 11:13:57,802 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 11:13:57,803 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 11:13:57,804 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 11:13:57,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,813 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 11:13:57,827 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,830 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,831 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,832 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,833 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,834 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,836 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,837 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,838 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,838 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,839 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 11:13:57,840 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,840 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 11:13:57,842 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 11:13:57,843 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 11:13:57,846 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 11:13:57,846 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 11:13:57,848 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 11:13:57,848 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 11:13:57,857 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 11:13:57,860 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 11:13:57,862 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 11:13:57,862 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 11:13:57,863 main DEBUG createAppenders(={Console}) 2024-12-05 11:13:57,864 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-05 11:13:57,864 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 11:13:57,865 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-05 11:13:57,865 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 11:13:57,866 main DEBUG OutputStream closed 2024-12-05 11:13:57,866 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 11:13:57,866 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 11:13:57,867 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-05 11:13:57,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 11:13:57,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 11:13:57,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 11:13:57,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 11:13:57,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 11:13:57,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 11:13:57,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 11:13:57,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 11:13:57,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 11:13:57,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 11:13:57,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 11:13:57,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 11:13:57,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 11:13:57,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 11:13:57,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 11:13:57,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 11:13:57,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 11:13:57,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 11:13:57,966 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 11:13:57,966 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-05 11:13:57,966 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 11:13:57,967 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-05T11:13:58,194 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a 2024-12-05 11:13:58,197 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 11:13:58,197 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T11:13:58,205 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-05T11:13:58,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T11:13:58,241 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61, deleteOnExit=true 2024-12-05T11:13:58,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T11:13:58,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/test.cache.data in system properties and HBase conf 2024-12-05T11:13:58,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T11:13:58,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/hadoop.log.dir in system properties and HBase conf 2024-12-05T11:13:58,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T11:13:58,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T11:13:58,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T11:13:58,326 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T11:13:58,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T11:13:58,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T11:13:58,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T11:13:58,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T11:13:58,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T11:13:58,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T11:13:58,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T11:13:58,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T11:13:58,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T11:13:58,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T11:13:58,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/nfs.dump.dir in system properties and HBase conf 2024-12-05T11:13:58,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/java.io.tmpdir in system properties and HBase conf 2024-12-05T11:13:58,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T11:13:58,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T11:13:58,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T11:13:59,571 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T11:13:59,636 INFO [Time-limited test {}] log.Log(170): Logging initialized @2456ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T11:13:59,697 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T11:13:59,751 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T11:13:59,770 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T11:13:59,770 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T11:13:59,771 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T11:13:59,783 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T11:13:59,785 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/hadoop.log.dir/,AVAILABLE} 2024-12-05T11:13:59,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T11:13:59,944 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f7463d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/java.io.tmpdir/jetty-localhost-38515-hadoop-hdfs-3_4_1-tests_jar-_-any-1274698871105315588/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T11:13:59,954 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:38515} 2024-12-05T11:13:59,955 INFO [Time-limited test {}] server.Server(415): Started @2775ms 2024-12-05T11:14:00,508 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T11:14:00,516 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T11:14:00,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T11:14:00,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T11:14:00,517 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T11:14:00,518 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/hadoop.log.dir/,AVAILABLE} 2024-12-05T11:14:00,519 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T11:14:00,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70061d7e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/java.io.tmpdir/jetty-localhost-46773-hadoop-hdfs-3_4_1-tests_jar-_-any-5220308546747222991/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T11:14:00,627 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:46773} 2024-12-05T11:14:00,627 INFO [Time-limited test {}] server.Server(415): Started @3447ms 2024-12-05T11:14:00,683 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T11:14:02,008 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/data/data2/current/BP-320430523-172.17.0.2-1733397238870/current, will proceed with Du for space computation calculation, 2024-12-05T11:14:02,008 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/data/data1/current/BP-320430523-172.17.0.2-1733397238870/current, will proceed with Du for space computation calculation, 2024-12-05T11:14:02,037 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T11:14:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ff4c9e7a8fdb1be with lease ID 0x3980ca3add33b583: Processing first storage report for DS-221e3b40-7407-4bdb-8efb-458ada629520 from datanode DatanodeRegistration(127.0.0.1:42527, datanodeUuid=bb5f54f6-9e96-4b7a-8f8a-b1bea9f6a6a2, infoPort=44459, infoSecurePort=0, ipcPort=44071, storageInfo=lv=-57;cid=testClusterID;nsid=1683585365;c=1733397238870) 2024-12-05T11:14:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ff4c9e7a8fdb1be with lease ID 0x3980ca3add33b583: from storage DS-221e3b40-7407-4bdb-8efb-458ada629520 node DatanodeRegistration(127.0.0.1:42527, datanodeUuid=bb5f54f6-9e96-4b7a-8f8a-b1bea9f6a6a2, infoPort=44459, infoSecurePort=0, ipcPort=44071, storageInfo=lv=-57;cid=testClusterID;nsid=1683585365;c=1733397238870), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-05T11:14:02,079 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ff4c9e7a8fdb1be with lease ID 0x3980ca3add33b583: Processing first storage report for DS-34bd47d5-8686-4b42-9eff-f264bae4825c from datanode DatanodeRegistration(127.0.0.1:42527, datanodeUuid=bb5f54f6-9e96-4b7a-8f8a-b1bea9f6a6a2, infoPort=44459, infoSecurePort=0, ipcPort=44071, storageInfo=lv=-57;cid=testClusterID;nsid=1683585365;c=1733397238870) 2024-12-05T11:14:02,079 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ff4c9e7a8fdb1be with lease ID 0x3980ca3add33b583: from storage DS-34bd47d5-8686-4b42-9eff-f264bae4825c node DatanodeRegistration(127.0.0.1:42527, datanodeUuid=bb5f54f6-9e96-4b7a-8f8a-b1bea9f6a6a2, infoPort=44459, infoSecurePort=0, ipcPort=44071, storageInfo=lv=-57;cid=testClusterID;nsid=1683585365;c=1733397238870), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T11:14:02,095 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a 2024-12-05T11:14:02,154 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/zookeeper_0, clientPort=50124, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T11:14:02,162 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50124 2024-12-05T11:14:02,170 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:02,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:02,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741825_1001 (size=7) 2024-12-05T11:14:02,762 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 with version=8 2024-12-05T11:14:02,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/hbase-staging 2024-12-05T11:14:02,839 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T11:14:03,075 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6706a97ae65f:0 server-side Connection retries=45 2024-12-05T11:14:03,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:14:03,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T11:14:03,087 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T11:14:03,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:14:03,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T11:14:03,200 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T11:14:03,249 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T11:14:03,256 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T11:14:03,260 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T11:14:03,281 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4731 (auto-detected) 2024-12-05T11:14:03,281 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T11:14:03,297 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38049 2024-12-05T11:14:03,315 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38049 connecting to ZooKeeper ensemble=127.0.0.1:50124 2024-12-05T11:14:03,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380490x0, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T11:14:03,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38049-0x101a532dcc70000 connected 2024-12-05T11:14:03,527 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:03,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:03,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:14:03,545 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5, hbase.cluster.distributed=false 2024-12-05T11:14:03,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T11:14:03,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38049 2024-12-05T11:14:03,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38049 2024-12-05T11:14:03,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38049 2024-12-05T11:14:03,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38049 2024-12-05T11:14:03,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38049 2024-12-05T11:14:03,669 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6706a97ae65f:0 server-side Connection retries=45 2024-12-05T11:14:03,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:14:03,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T11:14:03,671 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T11:14:03,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T11:14:03,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T11:14:03,673 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T11:14:03,676 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T11:14:03,677 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42931 2024-12-05T11:14:03,679 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42931 connecting to ZooKeeper ensemble=127.0.0.1:50124 2024-12-05T11:14:03,681 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:03,685 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:03,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429310x0, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T11:14:03,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:14:03,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42931-0x101a532dcc70001 connected 2024-12-05T11:14:03,703 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T11:14:03,709 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T11:14:03,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T11:14:03,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T11:14:03,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42931 2024-12-05T11:14:03,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42931 2024-12-05T11:14:03,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42931 2024-12-05T11:14:03,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42931 2024-12-05T11:14:03,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42931 2024-12-05T11:14:03,736 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6706a97ae65f:38049 2024-12-05T11:14:03,737 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6706a97ae65f,38049,1733397242931 2024-12-05T11:14:03,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:14:03,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:14:03,754 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6706a97ae65f,38049,1733397242931 2024-12-05T11:14:03,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:03,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T11:14:03,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:03,788 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T11:14:03,789 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6706a97ae65f,38049,1733397242931 from backup master directory 2024-12-05T11:14:03,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:14:03,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6706a97ae65f,38049,1733397242931 2024-12-05T11:14:03,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T11:14:03,798 WARN [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T11:14:03,799 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6706a97ae65f,38049,1733397242931 2024-12-05T11:14:03,801 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T11:14:03,802 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T11:14:03,853 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/hbase.id] with ID: 45b8d27a-9c74-4a66-9ada-fb2a65675e03 2024-12-05T11:14:03,853 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/.tmp/hbase.id 2024-12-05T11:14:03,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741826_1002 (size=42) 2024-12-05T11:14:04,269 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/.tmp/hbase.id]:[hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/hbase.id] 2024-12-05T11:14:04,322 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T11:14:04,326 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T11:14:04,344 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-05T11:14:04,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:04,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741827_1003 (size=196) 2024-12-05T11:14:04,841 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:14:04,843 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T11:14:04,856 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:04,860 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T11:14:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741828_1004 (size=1189) 2024-12-05T11:14:05,311 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store 2024-12-05T11:14:05,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741829_1005 (size=34) 2024-12-05T11:14:05,740 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T11:14:05,745 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:05,747 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T11:14:05,747 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:14:05,747 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:14:05,749 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T11:14:05,749 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:14:05,750 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:14:05,751 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733397245747Disabling compacts and flushes for region at 1733397245747Disabling writes for close at 1733397245749 (+2 ms)Writing region close event to WAL at 1733397245749Closed at 1733397245749 2024-12-05T11:14:05,753 WARN [master/6706a97ae65f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/.initializing 2024-12-05T11:14:05,753 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/WALs/6706a97ae65f,38049,1733397242931 2024-12-05T11:14:05,761 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T11:14:05,774 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6706a97ae65f%2C38049%2C1733397242931, suffix=, logDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/WALs/6706a97ae65f,38049,1733397242931, archiveDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/oldWALs, maxLogs=10 2024-12-05T11:14:05,804 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/WALs/6706a97ae65f,38049,1733397242931/6706a97ae65f%2C38049%2C1733397242931.1733397245779, exclude list is [], retry=0 2024-12-05T11:14:05,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42527,DS-221e3b40-7407-4bdb-8efb-458ada629520,DISK] 2024-12-05T11:14:05,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T11:14:05,852 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/WALs/6706a97ae65f,38049,1733397242931/6706a97ae65f%2C38049%2C1733397242931.1733397245779 2024-12-05T11:14:05,853 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459)] 2024-12-05T11:14:05,854 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:14:05,854 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:05,857 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,858 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T11:14:05,915 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:05,918 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:05,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T11:14:05,922 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:05,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:05,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T11:14:05,927 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:05,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:05,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,931 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T11:14:05,931 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:05,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:05,932 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,935 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,936 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,941 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,941 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,946 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T11:14:05,951 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T11:14:05,955 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:14:05,956 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67383883, jitterRate=0.004098102450370789}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T11:14:05,961 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733397245869Initializing all the Stores at 1733397245871 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397245872 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397245872Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397245873 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397245873Cleaning up temporary data from old regions at 1733397245942 (+69 ms)Region opened successfully at 1733397245961 (+19 ms) 2024-12-05T11:14:05,962 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T11:14:05,993 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd680fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6706a97ae65f/172.17.0.2:0 2024-12-05T11:14:06,022 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T11:14:06,031 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T11:14:06,031 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T11:14:06,033 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T11:14:06,034 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T11:14:06,038 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-05T11:14:06,038 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T11:14:06,065 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T11:14:06,074 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T11:14:06,218 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T11:14:06,221 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T11:14:06,222 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T11:14:06,235 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T11:14:06,237 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T11:14:06,242 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T11:14:06,249 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T11:14:06,251 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T11:14:06,260 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T11:14:06,283 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T11:14:06,291 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T11:14:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T11:14:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T11:14:06,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:06,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:06,306 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6706a97ae65f,38049,1733397242931, sessionid=0x101a532dcc70000, setting cluster-up flag (Was=false) 2024-12-05T11:14:06,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:06,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:06,376 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T11:14:06,379 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6706a97ae65f,38049,1733397242931 2024-12-05T11:14:06,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:06,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:06,440 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T11:14:06,445 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6706a97ae65f,38049,1733397242931 2024-12-05T11:14:06,453 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T11:14:06,515 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T11:14:06,523 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T11:14:06,528 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T11:14:06,530 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(746): ClusterId : 45b8d27a-9c74-4a66-9ada-fb2a65675e03 2024-12-05T11:14:06,532 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T11:14:06,533 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6706a97ae65f,38049,1733397242931 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T11:14:06,600 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T11:14:06,600 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T11:14:06,601 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6706a97ae65f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:14:06,601 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6706a97ae65f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:14:06,601 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6706a97ae65f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:14:06,602 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6706a97ae65f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T11:14:06,602 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6706a97ae65f:0, corePoolSize=10, maxPoolSize=10 2024-12-05T11:14:06,602 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,603 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6706a97ae65f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T11:14:06,603 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,606 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733397276605 2024-12-05T11:14:06,609 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T11:14:06,609 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T11:14:06,611 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T11:14:06,611 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T11:14:06,613 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T11:14:06,614 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T11:14:06,614 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T11:14:06,614 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T11:14:06,615 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,618 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T11:14:06,619 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:06,619 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T11:14:06,619 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T11:14:06,619 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T11:14:06,620 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T11:14:06,620 DEBUG [RS:0;6706a97ae65f:42931 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c455ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6706a97ae65f/172.17.0.2:0 2024-12-05T11:14:06,622 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T11:14:06,622 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T11:14:06,624 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6706a97ae65f:0:becomeActiveMaster-HFileCleaner.large.0-1733397246624,5,FailOnTimeoutGroup] 2024-12-05T11:14:06,625 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6706a97ae65f:0:becomeActiveMaster-HFileCleaner.small.0-1733397246625,5,FailOnTimeoutGroup] 2024-12-05T11:14:06,625 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,625 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T11:14:06,626 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,627 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741831_1007 (size=1321) 2024-12-05T11:14:06,633 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6706a97ae65f:42931 2024-12-05T11:14:06,636 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T11:14:06,636 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T11:14:06,636 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T11:14:06,638 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(2659): reportForDuty to master=6706a97ae65f,38049,1733397242931 with port=42931, startcode=1733397243639 2024-12-05T11:14:06,647 DEBUG [RS:0;6706a97ae65f:42931 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T11:14:06,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35471, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T11:14:06,677 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38049 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6706a97ae65f,42931,1733397243639 2024-12-05T11:14:06,679 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38049 {}] master.ServerManager(517): Registering regionserver=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:06,690 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:14:06,690 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35859 2024-12-05T11:14:06,690 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T11:14:06,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T11:14:06,703 DEBUG [RS:0;6706a97ae65f:42931 {}] zookeeper.ZKUtil(111): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6706a97ae65f,42931,1733397243639 2024-12-05T11:14:06,703 WARN [RS:0;6706a97ae65f:42931 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T11:14:06,703 INFO [RS:0;6706a97ae65f:42931 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T11:14:06,704 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639 2024-12-05T11:14:06,706 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6706a97ae65f,42931,1733397243639] 2024-12-05T11:14:06,729 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T11:14:06,740 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T11:14:06,744 INFO [RS:0;6706a97ae65f:42931 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T11:14:06,744 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,745 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T11:14:06,750 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T11:14:06,751 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,751 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,751 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,751 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6706a97ae65f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,752 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,753 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6706a97ae65f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T11:14:06,753 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6706a97ae65f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T11:14:06,753 DEBUG [RS:0;6706a97ae65f:42931 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T11:14:06,754 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,754 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,754 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,754 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,754 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,754 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,42931,1733397243639-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T11:14:06,769 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T11:14:06,771 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,42931,1733397243639-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,771 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,771 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.Replication(171): 6706a97ae65f,42931,1733397243639 started 2024-12-05T11:14:06,787 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:06,787 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1482): Serving as 6706a97ae65f,42931,1733397243639, RpcServer on 6706a97ae65f/172.17.0.2:42931, sessionid=0x101a532dcc70001 2024-12-05T11:14:06,788 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T11:14:06,788 DEBUG [RS:0;6706a97ae65f:42931 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6706a97ae65f,42931,1733397243639 2024-12-05T11:14:06,788 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6706a97ae65f,42931,1733397243639' 2024-12-05T11:14:06,788 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T11:14:06,789 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T11:14:06,790 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T11:14:06,790 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T11:14:06,790 DEBUG [RS:0;6706a97ae65f:42931 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6706a97ae65f,42931,1733397243639 2024-12-05T11:14:06,790 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6706a97ae65f,42931,1733397243639' 2024-12-05T11:14:06,790 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T11:14:06,791 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T11:14:06,792 DEBUG [RS:0;6706a97ae65f:42931 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T11:14:06,792 INFO [RS:0;6706a97ae65f:42931 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T11:14:06,792 INFO [RS:0;6706a97ae65f:42931 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T11:14:06,901 INFO [RS:0;6706a97ae65f:42931 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T11:14:06,906 INFO [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6706a97ae65f%2C42931%2C1733397243639, suffix=, logDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639, archiveDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/oldWALs, maxLogs=32 2024-12-05T11:14:06,920 DEBUG [RS:0;6706a97ae65f:42931 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639/6706a97ae65f%2C42931%2C1733397243639.1733397246909, exclude list is [], retry=0 2024-12-05T11:14:06,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42527,DS-221e3b40-7407-4bdb-8efb-458ada629520,DISK] 2024-12-05T11:14:06,927 INFO [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639/6706a97ae65f%2C42931%2C1733397243639.1733397246909 2024-12-05T11:14:06,928 DEBUG [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459)] 2024-12-05T11:14:07,034 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T11:14:07,035 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:14:07,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741833_1009 (size=32) 2024-12-05T11:14:07,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:07,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T11:14:07,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T11:14:07,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:07,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:07,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T11:14:07,465 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T11:14:07,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:07,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:07,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T11:14:07,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T11:14:07,470 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:07,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:07,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T11:14:07,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T11:14:07,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:07,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:07,474 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T11:14:07,476 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740 2024-12-05T11:14:07,476 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740 2024-12-05T11:14:07,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T11:14:07,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T11:14:07,480 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:14:07,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T11:14:07,485 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:14:07,486 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65230369, jitterRate=-0.027991756796836853}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:14:07,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733397247455Initializing all the Stores at 1733397247458 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397247458Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397247458Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397247458Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397247458Cleaning up temporary data from old regions at 1733397247479 (+21 ms)Region opened successfully at 1733397247487 (+8 ms) 2024-12-05T11:14:07,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T11:14:07,488 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T11:14:07,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T11:14:07,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T11:14:07,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T11:14:07,489 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T11:14:07,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733397247488Disabling compacts and flushes for region at 1733397247488Disabling writes for close at 1733397247488Writing region close event to WAL at 1733397247489 (+1 ms)Closed at 1733397247489 2024-12-05T11:14:07,492 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T11:14:07,492 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T11:14:07,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T11:14:07,504 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T11:14:07,507 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T11:14:07,663 DEBUG [6706a97ae65f:38049 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T11:14:07,678 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:07,682 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6706a97ae65f,42931,1733397243639, state=OPENING 2024-12-05T11:14:07,755 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T11:14:07,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:07,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:14:07,769 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:14:07,769 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:14:07,772 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T11:14:07,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:14:07,956 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T11:14:07,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57067, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T11:14:07,969 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T11:14:07,969 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T11:14:07,970 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T11:14:07,972 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6706a97ae65f%2C42931%2C1733397243639.meta, suffix=.meta, logDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639, archiveDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/oldWALs, maxLogs=32 2024-12-05T11:14:07,986 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639/6706a97ae65f%2C42931%2C1733397243639.meta.1733397247974.meta, exclude list is [], retry=0 2024-12-05T11:14:07,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42527,DS-221e3b40-7407-4bdb-8efb-458ada629520,DISK] 2024-12-05T11:14:07,992 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/WALs/6706a97ae65f,42931,1733397243639/6706a97ae65f%2C42931%2C1733397243639.meta.1733397247974.meta 2024-12-05T11:14:07,992 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459)] 2024-12-05T11:14:07,993 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:14:07,994 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T11:14:07,996 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T11:14:08,000 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T11:14:08,004 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T11:14:08,005 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:08,005 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T11:14:08,005 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T11:14:08,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T11:14:08,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T11:14:08,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:08,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:08,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T11:14:08,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T11:14:08,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:08,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:08,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T11:14:08,014 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T11:14:08,014 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:08,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:08,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T11:14:08,016 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T11:14:08,016 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:08,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T11:14:08,017 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T11:14:08,018 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740 2024-12-05T11:14:08,021 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740 2024-12-05T11:14:08,023 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T11:14:08,023 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T11:14:08,024 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:14:08,027 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T11:14:08,029 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73274354, jitterRate=0.09187296032905579}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:14:08,029 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T11:14:08,030 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733397248006Writing region info on filesystem at 1733397248006Initializing all the Stores at 1733397248007 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397248007Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397248008 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397248008Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733397248008Cleaning up temporary data from old regions at 1733397248023 (+15 ms)Running coprocessor post-open hooks at 1733397248029 (+6 ms)Region opened successfully at 1733397248030 (+1 ms) 2024-12-05T11:14:08,036 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733397247947 2024-12-05T11:14:08,046 DEBUG [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T11:14:08,046 INFO [RS_OPEN_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T11:14:08,048 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:08,050 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6706a97ae65f,42931,1733397243639, state=OPEN 2024-12-05T11:14:08,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T11:14:08,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T11:14:08,087 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:14:08,087 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T11:14:08,088 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:08,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T11:14:08,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6706a97ae65f,42931,1733397243639 in 313 msec 2024-12-05T11:14:08,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T11:14:08,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 603 msec 2024-12-05T11:14:08,106 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T11:14:08,106 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T11:14:08,125 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:08,126 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:08,150 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:08,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59473, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:08,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6880 sec 2024-12-05T11:14:08,172 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733397248171, completionTime=-1 2024-12-05T11:14:08,174 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T11:14:08,174 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T11:14:08,197 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T11:14:08,197 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733397308197 2024-12-05T11:14:08,197 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733397368197 2024-12-05T11:14:08,197 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-05T11:14:08,200 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,38049,1733397242931-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,200 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,38049,1733397242931-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,200 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,38049,1733397242931-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,202 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6706a97ae65f:38049, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,202 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,203 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,208 DEBUG [master/6706a97ae65f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T11:14:08,227 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.428sec 2024-12-05T11:14:08,228 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T11:14:08,229 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T11:14:08,229 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T11:14:08,230 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T11:14:08,230 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T11:14:08,230 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,38049,1733397242931-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T11:14:08,231 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,38049,1733397242931-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T11:14:08,238 DEBUG [master/6706a97ae65f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T11:14:08,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32936631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:08,239 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T11:14:08,239 INFO [master/6706a97ae65f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6706a97ae65f,38049,1733397242931-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T11:14:08,241 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T11:14:08,241 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T11:14:08,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:08,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:08,252 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:08,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:08,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60652350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:08,265 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:08,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:08,296 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6706a97ae65f,38049,1733397242931 2024-12-05T11:14:08,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=6737 2024-12-05T11:14:08,314 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:14:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:14:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@359a13b6 2024-12-05T11:14:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:14:08,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:14:08,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:14:08,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:14:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:08,354 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:14:08,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-05T11:14:08,357 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:08,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:14:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:14:08,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741835_1011 (size=1061) 2024-12-05T11:14:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:14:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:14:08,786 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:14:08,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741836_1012 (size=53) 2024-12-05T11:14:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:14:09,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:09,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fb2f3b651a249ac1bc6c3ae631d8f7b9, disabling compactions & flushes 2024-12-05T11:14:09,200 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,200 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. after waiting 0 ms 2024-12-05T11:14:09,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,201 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,201 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: Waiting for close lock at 1733397249200Disabling compacts and flushes for region at 1733397249200Disabling writes for close at 1733397249201 (+1 ms)Writing region close event to WAL at 1733397249201Closed at 1733397249201 2024-12-05T11:14:09,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:14:09,214 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733397249206"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733397249206"}]},"ts":"1733397249206"} 2024-12-05T11:14:09,218 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:14:09,220 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:14:09,222 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397249220"}]},"ts":"1733397249220"} 2024-12-05T11:14:09,226 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:14:09,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, ASSIGN}] 2024-12-05T11:14:09,230 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, ASSIGN 2024-12-05T11:14:09,232 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, ASSIGN; state=OFFLINE, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=false 2024-12-05T11:14:09,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb2f3b651a249ac1bc6c3ae631d8f7b9, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:09,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, ASSIGN because future has completed 2024-12-05T11:14:09,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:14:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:14:09,558 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,559 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:14:09,559 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,560 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:09,560 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,560 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,563 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,567 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:09,567 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb2f3b651a249ac1bc6c3ae631d8f7b9 columnFamilyName A 2024-12-05T11:14:09,567 DEBUG [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:09,568 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.HStore(327): Store=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:09,569 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,570 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:09,571 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb2f3b651a249ac1bc6c3ae631d8f7b9 columnFamilyName B 2024-12-05T11:14:09,571 DEBUG [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:09,572 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.HStore(327): Store=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:09,572 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,574 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:09,574 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb2f3b651a249ac1bc6c3ae631d8f7b9 columnFamilyName C 2024-12-05T11:14:09,574 DEBUG [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:09,575 INFO [StoreOpener-fb2f3b651a249ac1bc6c3ae631d8f7b9-1 {}] regionserver.HStore(327): Store=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:09,576 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,576 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,577 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,578 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,580 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,580 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,581 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:14:09,584 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,588 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:14:09,589 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fb2f3b651a249ac1bc6c3ae631d8f7b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65116925, jitterRate=-0.029682204127311707}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:14:09,589 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:09,590 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: Running coprocessor pre-open hook at 1733397249560Writing region info on filesystem at 1733397249560Initializing all the Stores at 1733397249562 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397249563 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397249563Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397249563Cleaning up temporary data from old regions at 1733397249580 (+17 ms)Running coprocessor post-open hooks at 1733397249589 (+9 ms)Region opened successfully at 1733397249590 (+1 ms) 2024-12-05T11:14:09,592 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., pid=6, masterSystemTime=1733397249547 2024-12-05T11:14:09,596 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,596 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:09,597 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb2f3b651a249ac1bc6c3ae631d8f7b9, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:09,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:14:09,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T11:14:09,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 in 213 msec 2024-12-05T11:14:09,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T11:14:09,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, ASSIGN in 381 msec 2024-12-05T11:14:09,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:14:09,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397249615"}]},"ts":"1733397249615"} 2024-12-05T11:14:09,618 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:14:09,620 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:14:09,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2730 sec 2024-12-05T11:14:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T11:14:10,522 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:10,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ec7dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,534 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea8828e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2b9a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,539 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:14:10,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:14:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a040aee 2024-12-05T11:14:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:14:10,551 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:14:10,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a862833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d5de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,562 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,566 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656f7adc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,569 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c642253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,576 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256affd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2401f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65800bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c15e47b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,591 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,594 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc1a8a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578c18ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1290d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,600 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175a1006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311faa9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a7f0f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,606 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46be8e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:10,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:10,609 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2268467d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:10,612 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f145b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:10,616 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,616 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,616 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,616 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,617 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,617 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,618 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,625 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,638 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-05T11:14:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:10,642 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:14:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:10,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:10,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:10,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:14:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:10,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:14:10,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:10,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:10,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:14:10,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/19886497ccf340db83c000c13519a694 is 50, key is test_row_0/A:col10/1733397250679/Put/seqid=0 2024-12-05T11:14:10,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:10,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:10,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:10,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:10,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:10,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397310810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:10,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=182, cellSize=8332, totalHeapSize=19168, min timestamp=1733397250792, max timestamp=1733397250828Num uniques 859; 2024-12-05T11:14:10,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397250792, max timestamp=1733397250828Num uniques 859; 2024-12-05T11:14:10,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397310810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:10,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397250792, max timestamp=1733397250828Num uniques 859; 2024-12-05T11:14:10,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T11:14:10,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8332, suffix heap size=117064,new segment heap size=19168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=182 2024-12-05T11:14:10,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741837_1013 (size=12001) 2024-12-05T11:14:10,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/19886497ccf340db83c000c13519a694 2024-12-05T11:14:10,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397310935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397310941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397310942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:14:10,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3267b8936dc64d15869def9f0689e12e is 50, key is test_row_0/B:col10/1733397250679/Put/seqid=0 2024-12-05T11:14:11,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397311019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397311020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741838_1014 (size=12001) 2024-12-05T11:14:11,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:11,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397311050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397311061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397311062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397311229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397311230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:14:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397311270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397311270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397311280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3267b8936dc64d15869def9f0689e12e 2024-12-05T11:14:11,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/c35f54ddf3bf4be4a3d1b872f127b392 is 50, key is test_row_0/C:col10/1733397250679/Put/seqid=0 2024-12-05T11:14:11,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741839_1015 (size=12001) 2024-12-05T11:14:11,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/c35f54ddf3bf4be4a3d1b872f127b392 2024-12-05T11:14:11,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:11,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/19886497ccf340db83c000c13519a694 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/19886497ccf340db83c000c13519a694 2024-12-05T11:14:11,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397311540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397311549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/19886497ccf340db83c000c13519a694, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T11:14:11,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3267b8936dc64d15869def9f0689e12e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3267b8936dc64d15869def9f0689e12e 2024-12-05T11:14:11,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3267b8936dc64d15869def9f0689e12e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T11:14:11,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/c35f54ddf3bf4be4a3d1b872f127b392 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/c35f54ddf3bf4be4a3d1b872f127b392 2024-12-05T11:14:11,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/c35f54ddf3bf4be4a3d1b872f127b392, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T11:14:11,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=121.40 KB/124313 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 907ms, sequenceid=13, compaction requested=false 2024-12-05T11:14:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T11:14:11,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:11,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=128.11 KB heapSize=366.48 KB 2024-12-05T11:14:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:11,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:11,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:11,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:11,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:11,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:11,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/d6ba06bc5b854b1d96487719cc3c529c is 50, key is test_row_0/A:col10/1733397250938/Put/seqid=0 2024-12-05T11:14:11,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741840_1016 (size=14341) 2024-12-05T11:14:11,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.52 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/d6ba06bc5b854b1d96487719cc3c529c 2024-12-05T11:14:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397311673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/94208a31c172407a93197715280c22c1 is 50, key is test_row_0/B:col10/1733397250938/Put/seqid=0 2024-12-05T11:14:11,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741841_1017 (size=12001) 2024-12-05T11:14:11,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/94208a31c172407a93197715280c22c1 2024-12-05T11:14:11,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397311700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397311707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6f90d99d983c4ba7bac46776bdf3fbf9 is 50, key is test_row_0/C:col10/1733397250938/Put/seqid=0 2024-12-05T11:14:11,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741842_1018 (size=12001) 2024-12-05T11:14:11,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6f90d99d983c4ba7bac46776bdf3fbf9 2024-12-05T11:14:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:14:11,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/d6ba06bc5b854b1d96487719cc3c529c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d6ba06bc5b854b1d96487719cc3c529c 2024-12-05T11:14:11,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d6ba06bc5b854b1d96487719cc3c529c, entries=200, sequenceid=49, filesize=14.0 K 2024-12-05T11:14:11,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/94208a31c172407a93197715280c22c1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/94208a31c172407a93197715280c22c1 2024-12-05T11:14:11,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:11,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/94208a31c172407a93197715280c22c1, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T11:14:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6f90d99d983c4ba7bac46776bdf3fbf9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6f90d99d983c4ba7bac46776bdf3fbf9 2024-12-05T11:14:11,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397311819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397311839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6f90d99d983c4ba7bac46776bdf3fbf9, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T11:14:11,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397311839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:11,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~128.11 KB/131183, heapSize ~366.44 KB/375232, currentSize=53.67 KB/54960 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 233ms, sequenceid=49, compaction requested=false 2024-12-05T11:14:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:11,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T11:14:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:11,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:14:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/0ac0d59b0bf2468d8ba6e715fda2e845 is 50, key is test_row_0/A:col10/1733397251650/Put/seqid=0 2024-12-05T11:14:12,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741843_1019 (size=12001) 2024-12-05T11:14:12,026 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/0ac0d59b0bf2468d8ba6e715fda2e845 2024-12-05T11:14:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:12,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/e3c04b231d1448b38083ce14fd74573e is 50, key is test_row_0/B:col10/1733397251650/Put/seqid=0 2024-12-05T11:14:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741844_1020 (size=12001) 2024-12-05T11:14:12,085 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/e3c04b231d1448b38083ce14fd74573e 2024-12-05T11:14:12,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/bd1f091ffcdc4d35b7c5c53ce782a182 is 50, key is test_row_0/C:col10/1733397251650/Put/seqid=0 2024-12-05T11:14:12,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397252065, max timestamp=1733397252112Num uniques 859; 2024-12-05T11:14:12,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:12,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741845_1021 (size=12001) 2024-12-05T11:14:12,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397252065, max timestamp=1733397252112Num uniques 859; 2024-12-05T11:14:12,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:12,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397252065, max timestamp=1733397252119Num uniques 859; 2024-12-05T11:14:12,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:12,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397312163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397312166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397312166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397312172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397312176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397312289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397312289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397312290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397312290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397312290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397312499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397312499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397312500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397312500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397312500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,527 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/bd1f091ffcdc4d35b7c5c53ce782a182 2024-12-05T11:14:12,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/0ac0d59b0bf2468d8ba6e715fda2e845 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/0ac0d59b0bf2468d8ba6e715fda2e845 2024-12-05T11:14:12,559 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/0ac0d59b0bf2468d8ba6e715fda2e845, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:14:12,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/e3c04b231d1448b38083ce14fd74573e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/e3c04b231d1448b38083ce14fd74573e 2024-12-05T11:14:12,581 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/e3c04b231d1448b38083ce14fd74573e, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:14:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/bd1f091ffcdc4d35b7c5c53ce782a182 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/bd1f091ffcdc4d35b7c5c53ce782a182 2024-12-05T11:14:12,603 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/bd1f091ffcdc4d35b7c5c53ce782a182, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:14:12,606 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 628ms, sequenceid=60, compaction requested=true 2024-12-05T11:14:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:12,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-05T11:14:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-05T11:14:12,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T11:14:12,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-12-05T11:14:12,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9870 sec 2024-12-05T11:14:12,724 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T11:14:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T11:14:12,800 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:12,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-05T11:14:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:14:12,806 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:12,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:12,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:12,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T11:14:12,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:12,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:12,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/5bee48d0a1f140f9b0c5573a0786798a is 50, key is test_row_0/A:col10/1733397252151/Put/seqid=0 2024-12-05T11:14:12,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397312821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397312829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397312832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397312835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397312837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741846_1022 (size=14341) 2024-12-05T11:14:12,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/5bee48d0a1f140f9b0c5573a0786798a 2024-12-05T11:14:12,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/c84567d97fa54030b07f6f93936f86bd is 50, key is test_row_0/B:col10/1733397252151/Put/seqid=0 2024-12-05T11:14:12,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741847_1023 (size=12001) 2024-12-05T11:14:12,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/c84567d97fa54030b07f6f93936f86bd 2024-12-05T11:14:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:14:12,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/f19b78acea92496888934ac52f91e369 is 50, key is test_row_0/C:col10/1733397252151/Put/seqid=0 2024-12-05T11:14:12,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397312939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741848_1024 (size=12001) 2024-12-05T11:14:12,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/f19b78acea92496888934ac52f91e369 2024-12-05T11:14:12,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397312950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397312951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397312951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:12,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397312951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:12,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T11:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:12,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:12,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/5bee48d0a1f140f9b0c5573a0786798a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5bee48d0a1f140f9b0c5573a0786798a 2024-12-05T11:14:12,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5bee48d0a1f140f9b0c5573a0786798a, entries=200, sequenceid=97, filesize=14.0 K 2024-12-05T11:14:12,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/c84567d97fa54030b07f6f93936f86bd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/c84567d97fa54030b07f6f93936f86bd 2024-12-05T11:14:13,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/c84567d97fa54030b07f6f93936f86bd, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T11:14:13,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/f19b78acea92496888934ac52f91e369 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f19b78acea92496888934ac52f91e369 2024-12-05T11:14:13,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f19b78acea92496888934ac52f91e369, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T11:14:13,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 225ms, sequenceid=97, compaction requested=true 2024-12-05T11:14:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:13,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:13,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:13,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:13,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:13,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:13,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:13,050 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:13,050 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:13,058 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:13,060 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:13,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:13,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:13,061 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:13,061 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3267b8936dc64d15869def9f0689e12e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/94208a31c172407a93197715280c22c1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/e3c04b231d1448b38083ce14fd74573e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/c84567d97fa54030b07f6f93936f86bd] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=46.9 K 2024-12-05T11:14:13,064 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3267b8936dc64d15869def9f0689e12e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733397250677 2024-12-05T11:14:13,065 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:13,065 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/19886497ccf340db83c000c13519a694, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d6ba06bc5b854b1d96487719cc3c529c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/0ac0d59b0bf2468d8ba6e715fda2e845, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5bee48d0a1f140f9b0c5573a0786798a] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=51.4 K 2024-12-05T11:14:13,066 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 94208a31c172407a93197715280c22c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733397250934 2024-12-05T11:14:13,066 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19886497ccf340db83c000c13519a694, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733397250677 2024-12-05T11:14:13,067 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c04b231d1448b38083ce14fd74573e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397251650 2024-12-05T11:14:13,067 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6ba06bc5b854b1d96487719cc3c529c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733397250925 2024-12-05T11:14:13,068 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c84567d97fa54030b07f6f93936f86bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397252151 2024-12-05T11:14:13,070 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac0d59b0bf2468d8ba6e715fda2e845, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397251650 2024-12-05T11:14:13,078 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bee48d0a1f140f9b0c5573a0786798a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397252151 2024-12-05T11:14:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:14:13,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T11:14:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:13,120 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:14:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:13,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7357e62b89ae409e9fee18a798cffd5b is 50, key is test_row_0/A:col10/1733397252831/Put/seqid=0 2024-12-05T11:14:13,153 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#13 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:13,154 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/b2ea9e60c9fb4075946d256b851a94a2 is 50, key is test_row_0/A:col10/1733397252151/Put/seqid=0 2024-12-05T11:14:13,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:13,159 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:13,160 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/d1b24ed89e094643b010f0bb0d3d74d4 is 50, key is test_row_0/B:col10/1733397252151/Put/seqid=0 2024-12-05T11:14:13,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741849_1025 (size=12139) 2024-12-05T11:14:13,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741851_1027 (size=12001) 2024-12-05T11:14:13,215 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/b2ea9e60c9fb4075946d256b851a94a2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/b2ea9e60c9fb4075946d256b851a94a2 2024-12-05T11:14:13,217 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7357e62b89ae409e9fee18a798cffd5b 2024-12-05T11:14:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741850_1026 (size=12139) 2024-12-05T11:14:13,239 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into b2ea9e60c9fb4075946d256b851a94a2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:13,239 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:13,241 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=12, startTime=1733397253039; duration=0sec 2024-12-05T11:14:13,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/2fdf6eec56b8403a86564a19a4d2ace0 is 50, key is test_row_0/B:col10/1733397252831/Put/seqid=0 2024-12-05T11:14:13,242 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:13,242 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:13,242 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:13,246 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/d1b24ed89e094643b010f0bb0d3d74d4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d1b24ed89e094643b010f0bb0d3d74d4 2024-12-05T11:14:13,249 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:13,249 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:13,249 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:13,250 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/c35f54ddf3bf4be4a3d1b872f127b392, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6f90d99d983c4ba7bac46776bdf3fbf9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/bd1f091ffcdc4d35b7c5c53ce782a182, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f19b78acea92496888934ac52f91e369] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=46.9 K 2024-12-05T11:14:13,251 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35f54ddf3bf4be4a3d1b872f127b392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733397250677 2024-12-05T11:14:13,252 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f90d99d983c4ba7bac46776bdf3fbf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733397250934 2024-12-05T11:14:13,254 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd1f091ffcdc4d35b7c5c53ce782a182, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397251650 2024-12-05T11:14:13,255 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f19b78acea92496888934ac52f91e369, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397252151 2024-12-05T11:14:13,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741852_1028 (size=12001) 2024-12-05T11:14:13,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397253180, max timestamp=1733397253247Num uniques 859; 2024-12-05T11:14:13,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:13,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397253180, max timestamp=1733397253249Num uniques 859; 2024-12-05T11:14:13,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T11:14:13,267 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/2fdf6eec56b8403a86564a19a4d2ace0 2024-12-05T11:14:13,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397253180, max timestamp=1733397253260Num uniques 859; 2024-12-05T11:14:13,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:13,273 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into d1b24ed89e094643b010f0bb0d3d74d4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:13,273 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:13,273 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=12, startTime=1733397253047; duration=0sec 2024-12-05T11:14:13,274 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:13,274 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:13,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/65846ace328a4368adc8fef699d53719 is 50, key is test_row_0/C:col10/1733397252831/Put/seqid=0 2024-12-05T11:14:13,292 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#17 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:13,293 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/fe1d5b08d4584399b6598de799422b8f is 50, key is test_row_0/C:col10/1733397252151/Put/seqid=0 2024-12-05T11:14:13,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741853_1029 (size=12001) 2024-12-05T11:14:13,313 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/65846ace328a4368adc8fef699d53719 2024-12-05T11:14:13,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397313308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397313308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7357e62b89ae409e9fee18a798cffd5b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7357e62b89ae409e9fee18a798cffd5b 2024-12-05T11:14:13,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733397253301, max timestamp=1733397253315Num uniques 1009; 2024-12-05T11:14:13,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733397253301, max timestamp=1733397253315Num uniques 1018; 2024-12-05T11:14:13,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-12-05T11:14:13,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-05T11:14:13,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741854_1030 (size=12139) 2024-12-05T11:14:13,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397253304, max timestamp=1733397253315Num uniques 1018; 2024-12-05T11:14:13,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:14:13,352 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/fe1d5b08d4584399b6598de799422b8f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/fe1d5b08d4584399b6598de799422b8f 2024-12-05T11:14:13,366 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7357e62b89ae409e9fee18a798cffd5b, entries=150, sequenceid=107, filesize=11.7 K 2024-12-05T11:14:13,370 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into fe1d5b08d4584399b6598de799422b8f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:13,370 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:13,370 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=12, startTime=1733397253047; duration=0sec 2024-12-05T11:14:13,370 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:13,370 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/2fdf6eec56b8403a86564a19a4d2ace0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/2fdf6eec56b8403a86564a19a4d2ace0 2024-12-05T11:14:13,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/2fdf6eec56b8403a86564a19a4d2ace0, entries=150, sequenceid=107, filesize=11.7 K 2024-12-05T11:14:13,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397313392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397313393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397313396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/65846ace328a4368adc8fef699d53719 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/65846ace328a4368adc8fef699d53719 2024-12-05T11:14:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,414 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/65846ace328a4368adc8fef699d53719, entries=150, sequenceid=107, filesize=11.7 K 2024-12-05T11:14:13,417 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 297ms, sequenceid=107, compaction requested=false 2024-12-05T11:14:13,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T11:14:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-05T11:14:13,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T11:14:13,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 611 msec 2024-12-05T11:14:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:14:13,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 628 msec 2024-12-05T11:14:13,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=143.13 KB heapSize=404.69 KB 2024-12-05T11:14:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:13,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733397253361, max timestamp=1733397253433Num uniques 1027; 2024-12-05T11:14:13,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-05T11:14:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:13,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:14:13,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:13,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:14:13,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/532cb9bae22d482aaacf2da7e69a3283 is 50, key is test_row_0/A:col10/1733397253433/Put/seqid=0 2024-12-05T11:14:13,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741855_1031 (size=16931) 2024-12-05T11:14:13,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/532cb9bae22d482aaacf2da7e69a3283 2024-12-05T11:14:13,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397313486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397313497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/7d997a2b75d2401e963942008d0aac9c is 50, key is test_row_0/B:col10/1733397253433/Put/seqid=0 2024-12-05T11:14:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397313509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397313509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397313509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,520 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:14:13,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741856_1032 (size=12151) 2024-12-05T11:14:13,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T11:14:13,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T11:14:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397313609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T11:14:13,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T11:14:13,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T11:14:13,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T11:14:13,621 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T11:14:13,622 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T11:14:13,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397313609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397313718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397313728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397313729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397313829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397313829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T11:14:13,939 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:13,939 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-05T11:14:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:13,944 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:13,945 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:13,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:13,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/7d997a2b75d2401e963942008d0aac9c 2024-12-05T11:14:13,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/e280993fa7294102b559426f0a4b9c60 is 50, key is test_row_0/C:col10/1733397253433/Put/seqid=0 2024-12-05T11:14:13,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741857_1033 (size=12151) 2024-12-05T11:14:14,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397314029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397314038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397314038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:14,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:14,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:14,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397314139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397314139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:14,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:14,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/e280993fa7294102b559426f0a4b9c60 2024-12-05T11:14:14,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/532cb9bae22d482aaacf2da7e69a3283 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/532cb9bae22d482aaacf2da7e69a3283 2024-12-05T11:14:14,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:14,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:14,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/532cb9bae22d482aaacf2da7e69a3283, entries=250, sequenceid=165, filesize=16.5 K 2024-12-05T11:14:14,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/7d997a2b75d2401e963942008d0aac9c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7d997a2b75d2401e963942008d0aac9c 2024-12-05T11:14:14,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7d997a2b75d2401e963942008d0aac9c, entries=150, sequenceid=165, filesize=11.9 K 2024-12-05T11:14:14,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/e280993fa7294102b559426f0a4b9c60 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e280993fa7294102b559426f0a4b9c60 2024-12-05T11:14:14,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e280993fa7294102b559426f0a4b9c60, entries=150, sequenceid=165, filesize=11.9 K 2024-12-05T11:14:14,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~316.70 KB/324304, currentSize=67.09 KB/68700 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1020ms, sequenceid=165, compaction requested=true 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:14,457 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:14,457 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:14,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:14,459 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:14,459 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41071 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:14,459 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:14,459 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:14,459 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,459 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,459 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d1b24ed89e094643b010f0bb0d3d74d4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/2fdf6eec56b8403a86564a19a4d2ace0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7d997a2b75d2401e963942008d0aac9c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=35.4 K 2024-12-05T11:14:14,459 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/b2ea9e60c9fb4075946d256b851a94a2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7357e62b89ae409e9fee18a798cffd5b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/532cb9bae22d482aaacf2da7e69a3283] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=40.1 K 2024-12-05T11:14:14,460 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b24ed89e094643b010f0bb0d3d74d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397252151 2024-12-05T11:14:14,461 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ea9e60c9fb4075946d256b851a94a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397252151 2024-12-05T11:14:14,461 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fdf6eec56b8403a86564a19a4d2ace0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733397252827 2024-12-05T11:14:14,462 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7357e62b89ae409e9fee18a798cffd5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733397252827 2024-12-05T11:14:14,463 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d997a2b75d2401e963942008d0aac9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733397253367 2024-12-05T11:14:14,466 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 532cb9bae22d482aaacf2da7e69a3283, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733397253361 2024-12-05T11:14:14,493 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:14,494 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/8c7c12e22fd742cab2716f677dab18bb is 50, key is test_row_0/B:col10/1733397253433/Put/seqid=0 2024-12-05T11:14:14,499 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:14,500 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/453b47ce97aa45619fefed595c0721f8 is 50, key is test_row_0/A:col10/1733397253433/Put/seqid=0 2024-12-05T11:14:14,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741858_1034 (size=12391) 2024-12-05T11:14:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:14,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:14:14,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:14,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:14,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:14,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:14,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:14,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:14,547 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/8c7c12e22fd742cab2716f677dab18bb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/8c7c12e22fd742cab2716f677dab18bb 2024-12-05T11:14:14,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741859_1035 (size=12391) 2024-12-05T11:14:14,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/2b012d4031f4441ea11f0a78dcd8be61 is 50, key is test_row_0/A:col10/1733397253491/Put/seqid=0 2024-12-05T11:14:14,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,574 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/453b47ce97aa45619fefed595c0721f8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/453b47ce97aa45619fefed595c0721f8 2024-12-05T11:14:14,579 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 8c7c12e22fd742cab2716f677dab18bb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:14,580 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:14,581 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=13, startTime=1733397254457; duration=0sec 2024-12-05T11:14:14,583 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:14,584 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:14,584 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:14,586 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:14,586 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:14,586 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,586 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/fe1d5b08d4584399b6598de799422b8f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/65846ace328a4368adc8fef699d53719, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e280993fa7294102b559426f0a4b9c60] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=35.4 K 2024-12-05T11:14:14,588 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting fe1d5b08d4584399b6598de799422b8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397252151 2024-12-05T11:14:14,588 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 453b47ce97aa45619fefed595c0721f8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:14,589 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:14,589 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=13, startTime=1733397254457; duration=0sec 2024-12-05T11:14:14,589 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:14,589 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:14,589 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 65846ace328a4368adc8fef699d53719, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733397252827 2024-12-05T11:14:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741860_1036 (size=12151) 2024-12-05T11:14:14,590 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e280993fa7294102b559426f0a4b9c60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733397253367 2024-12-05T11:14:14,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/2b012d4031f4441ea11f0a78dcd8be61 2024-12-05T11:14:14,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/6c87f4807c4d412baa89edc457993c40 is 50, key is test_row_0/B:col10/1733397253491/Put/seqid=0 2024-12-05T11:14:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,611 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#25 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397314609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,612 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/efc579fc8fdc4c77bfc159d803a64845 is 50, key is test_row_0/C:col10/1733397253433/Put/seqid=0 2024-12-05T11:14:14,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397314610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397314611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741861_1037 (size=12151) 2024-12-05T11:14:14,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/6c87f4807c4d412baa89edc457993c40 2024-12-05T11:14:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397314649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/2e7572b991184531afb776f8a4d22051 is 50, key is test_row_0/C:col10/1733397253491/Put/seqid=0 2024-12-05T11:14:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741862_1038 (size=12391) 2024-12-05T11:14:14,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397314659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741863_1039 (size=12151) 2024-12-05T11:14:14,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397314719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:14,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:14,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397314729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397314729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:14,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:14,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397314928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397314939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397314939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:15,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:15,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:15,072 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/efc579fc8fdc4c77bfc159d803a64845 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/efc579fc8fdc4c77bfc159d803a64845 2024-12-05T11:14:15,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/2e7572b991184531afb776f8a4d22051 2024-12-05T11:14:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:15,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/2b012d4031f4441ea11f0a78dcd8be61 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/2b012d4031f4441ea11f0a78dcd8be61 2024-12-05T11:14:15,101 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into efc579fc8fdc4c77bfc159d803a64845(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:15,101 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:15,101 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=13, startTime=1733397254457; duration=0sec 2024-12-05T11:14:15,101 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:15,102 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:15,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/2b012d4031f4441ea11f0a78dcd8be61, entries=150, sequenceid=179, filesize=11.9 K 2024-12-05T11:14:15,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/6c87f4807c4d412baa89edc457993c40 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/6c87f4807c4d412baa89edc457993c40 2024-12-05T11:14:15,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/6c87f4807c4d412baa89edc457993c40, entries=150, sequenceid=179, filesize=11.9 K 2024-12-05T11:14:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/2e7572b991184531afb776f8a4d22051 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/2e7572b991184531afb776f8a4d22051 2024-12-05T11:14:15,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/2e7572b991184531afb776f8a4d22051, entries=150, sequenceid=179, filesize=11.9 K 2024-12-05T11:14:15,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 595ms, sequenceid=179, compaction requested=false 2024-12-05T11:14:15,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:15,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T11:14:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:15,193 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:14:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:15,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/26491dd4567942eba728c91f5638c76d is 50, key is test_row_0/A:col10/1733397254607/Put/seqid=0 2024-12-05T11:14:15,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741864_1040 (size=12151) 2024-12-05T11:14:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:15,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397315260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397315263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397315265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397315369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397315379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397315379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397315579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397315588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397315588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,634 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/26491dd4567942eba728c91f5638c76d 2024-12-05T11:14:15,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1d8ef90e065d42379df802f49561865f is 50, key is test_row_0/B:col10/1733397254607/Put/seqid=0 2024-12-05T11:14:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397315660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,665 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397315680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,688 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741865_1041 (size=12151) 2024-12-05T11:14:15,694 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1d8ef90e065d42379df802f49561865f 2024-12-05T11:14:15,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ddf951124c16440497265ac503673536 is 50, key is test_row_0/C:col10/1733397254607/Put/seqid=0 2024-12-05T11:14:15,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741866_1042 (size=12151) 2024-12-05T11:14:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397315889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397315898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:15,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397315909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:15,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:16,133 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ddf951124c16440497265ac503673536 2024-12-05T11:14:16,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/26491dd4567942eba728c91f5638c76d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/26491dd4567942eba728c91f5638c76d 2024-12-05T11:14:16,171 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/26491dd4567942eba728c91f5638c76d, entries=150, sequenceid=201, filesize=11.9 K 2024-12-05T11:14:16,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1d8ef90e065d42379df802f49561865f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1d8ef90e065d42379df802f49561865f 2024-12-05T11:14:16,195 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1d8ef90e065d42379df802f49561865f, entries=150, sequenceid=201, filesize=11.9 K 2024-12-05T11:14:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ddf951124c16440497265ac503673536 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ddf951124c16440497265ac503673536 2024-12-05T11:14:16,214 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ddf951124c16440497265ac503673536, entries=150, sequenceid=201, filesize=11.9 K 2024-12-05T11:14:16,216 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1022ms, sequenceid=201, compaction requested=true 2024-12-05T11:14:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T11:14:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-05T11:14:16,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T11:14:16,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2730 sec 2024-12-05T11:14:16,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.2830 sec 2024-12-05T11:14:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:16,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:14:16,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:16,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:16,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:16,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:16,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:16,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:16,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7a5f3a1397bf4cb1aee7010eb44edf2a is 50, key is test_row_0/A:col10/1733397256401/Put/seqid=0 2024-12-05T11:14:16,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741867_1043 (size=12151) 2024-12-05T11:14:16,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7a5f3a1397bf4cb1aee7010eb44edf2a 2024-12-05T11:14:16,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3e31ec38330d4d758ca707f657725456 is 50, key is test_row_0/B:col10/1733397256401/Put/seqid=0 2024-12-05T11:14:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397316463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397316465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397316463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741868_1044 (size=12151) 2024-12-05T11:14:16,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3e31ec38330d4d758ca707f657725456 2024-12-05T11:14:16,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/e8340ea339e24dee9c65aa25a9dfbfe8 is 50, key is test_row_0/C:col10/1733397256401/Put/seqid=0 2024-12-05T11:14:16,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741869_1045 (size=12151) 2024-12-05T11:14:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397316798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397316799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397316799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:16,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/e8340ea339e24dee9c65aa25a9dfbfe8 2024-12-05T11:14:17,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7a5f3a1397bf4cb1aee7010eb44edf2a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a5f3a1397bf4cb1aee7010eb44edf2a 2024-12-05T11:14:17,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a5f3a1397bf4cb1aee7010eb44edf2a, entries=150, sequenceid=215, filesize=11.9 K 2024-12-05T11:14:17,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3e31ec38330d4d758ca707f657725456 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3e31ec38330d4d758ca707f657725456 2024-12-05T11:14:17,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3e31ec38330d4d758ca707f657725456, entries=150, sequenceid=215, filesize=11.9 K 2024-12-05T11:14:17,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/e8340ea339e24dee9c65aa25a9dfbfe8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e8340ea339e24dee9c65aa25a9dfbfe8 2024-12-05T11:14:17,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e8340ea339e24dee9c65aa25a9dfbfe8, entries=150, sequenceid=215, filesize=11.9 K 2024-12-05T11:14:17,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 647ms, sequenceid=215, compaction requested=true 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:17,052 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:17,052 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:17,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:17,054 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:17,054 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:17,054 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:17,054 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:17,054 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:17,054 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:17,055 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/8c7c12e22fd742cab2716f677dab18bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/6c87f4807c4d412baa89edc457993c40, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1d8ef90e065d42379df802f49561865f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3e31ec38330d4d758ca707f657725456] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=47.7 K 2024-12-05T11:14:17,055 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/453b47ce97aa45619fefed595c0721f8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/2b012d4031f4441ea11f0a78dcd8be61, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/26491dd4567942eba728c91f5638c76d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a5f3a1397bf4cb1aee7010eb44edf2a] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=47.7 K 2024-12-05T11:14:17,055 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 453b47ce97aa45619fefed595c0721f8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733397253367 2024-12-05T11:14:17,056 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7c12e22fd742cab2716f677dab18bb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733397253367 2024-12-05T11:14:17,056 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c87f4807c4d412baa89edc457993c40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733397253460 2024-12-05T11:14:17,057 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b012d4031f4441ea11f0a78dcd8be61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733397253460 2024-12-05T11:14:17,057 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26491dd4567942eba728c91f5638c76d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733397254600 2024-12-05T11:14:17,058 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8ef90e065d42379df802f49561865f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733397254600 2024-12-05T11:14:17,058 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e31ec38330d4d758ca707f657725456, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733397255257 2024-12-05T11:14:17,059 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a5f3a1397bf4cb1aee7010eb44edf2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733397255257 2024-12-05T11:14:17,099 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:17,102 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/b5012ec1881440f6b005d9d0dbf73cee is 50, key is test_row_0/B:col10/1733397256401/Put/seqid=0 2024-12-05T11:14:17,104 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#34 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:17,105 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/bd516ed0bc434089a6d5a435a29764e3 is 50, key is test_row_0/A:col10/1733397256401/Put/seqid=0 2024-12-05T11:14:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:17,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T11:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:17,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397317138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741870_1046 (size=12527) 2024-12-05T11:14:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397317141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397317142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,155 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/b5012ec1881440f6b005d9d0dbf73cee as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b5012ec1881440f6b005d9d0dbf73cee 2024-12-05T11:14:17,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/62940dad079944818cbf8a18cc05030d is 50, key is test_row_0/A:col10/1733397256444/Put/seqid=0 2024-12-05T11:14:17,168 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into b5012ec1881440f6b005d9d0dbf73cee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:17,168 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:17,168 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=12, startTime=1733397257052; duration=0sec 2024-12-05T11:14:17,171 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:17,171 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:17,172 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:17,177 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:17,178 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:17,179 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:17,179 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/efc579fc8fdc4c77bfc159d803a64845, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/2e7572b991184531afb776f8a4d22051, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ddf951124c16440497265ac503673536, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e8340ea339e24dee9c65aa25a9dfbfe8] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=47.7 K 2024-12-05T11:14:17,180 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting efc579fc8fdc4c77bfc159d803a64845, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733397253367 2024-12-05T11:14:17,180 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e7572b991184531afb776f8a4d22051, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733397253460 2024-12-05T11:14:17,181 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf951124c16440497265ac503673536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733397254600 2024-12-05T11:14:17,181 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e8340ea339e24dee9c65aa25a9dfbfe8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733397255257 2024-12-05T11:14:17,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741871_1047 (size=12527) 2024-12-05T11:14:17,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741872_1048 (size=14541) 2024-12-05T11:14:17,215 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:17,216 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/4613964f5e8c42129e5cf8d096496741 is 50, key is test_row_0/C:col10/1733397256401/Put/seqid=0 2024-12-05T11:14:17,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741873_1049 (size=12527) 2024-12-05T11:14:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397317248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,254 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/4613964f5e8c42129e5cf8d096496741 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/4613964f5e8c42129e5cf8d096496741 2024-12-05T11:14:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397317259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397317260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,273 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 4613964f5e8c42129e5cf8d096496741(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:17,273 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:17,273 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=12, startTime=1733397257052; duration=0sec 2024-12-05T11:14:17,274 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:17,274 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:17,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397317459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397317469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397317469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,599 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/bd516ed0bc434089a6d5a435a29764e3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/bd516ed0bc434089a6d5a435a29764e3 2024-12-05T11:14:17,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/62940dad079944818cbf8a18cc05030d 2024-12-05T11:14:17,616 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into bd516ed0bc434089a6d5a435a29764e3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:17,616 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:17,616 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=12, startTime=1733397257052; duration=0sec 2024-12-05T11:14:17,620 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:17,620 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:17,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/18023e32b3934313b0999eff0345d3f9 is 50, key is test_row_0/B:col10/1733397256444/Put/seqid=0 2024-12-05T11:14:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741874_1050 (size=12151) 2024-12-05T11:14:17,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/18023e32b3934313b0999eff0345d3f9 2024-12-05T11:14:17,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/23b72e11e17b4d5faa9d9ee766d1aa9c is 50, key is test_row_0/C:col10/1733397256444/Put/seqid=0 2024-12-05T11:14:17,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741875_1051 (size=12151) 2024-12-05T11:14:17,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/23b72e11e17b4d5faa9d9ee766d1aa9c 2024-12-05T11:14:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397317679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,683 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/62940dad079944818cbf8a18cc05030d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/62940dad079944818cbf8a18cc05030d 2024-12-05T11:14:17,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/62940dad079944818cbf8a18cc05030d, entries=200, sequenceid=236, filesize=14.2 K 2024-12-05T11:14:17,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/18023e32b3934313b0999eff0345d3f9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/18023e32b3934313b0999eff0345d3f9 2024-12-05T11:14:17,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397317698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,701 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/18023e32b3934313b0999eff0345d3f9, entries=150, sequenceid=236, filesize=11.9 K 2024-12-05T11:14:17,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/23b72e11e17b4d5faa9d9ee766d1aa9c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/23b72e11e17b4d5faa9d9ee766d1aa9c 2024-12-05T11:14:17,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/23b72e11e17b4d5faa9d9ee766d1aa9c, entries=150, sequenceid=236, filesize=11.9 K 2024-12-05T11:14:17,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 608ms, sequenceid=236, compaction requested=false 2024-12-05T11:14:17,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:17,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T11:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:17,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:17,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/222c1d0b81a6404c9b5f14b9abc5f79b is 50, key is test_row_0/A:col10/1733397257137/Put/seqid=0 2024-12-05T11:14:17,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741876_1052 (size=12151) 2024-12-05T11:14:17,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/222c1d0b81a6404c9b5f14b9abc5f79b 2024-12-05T11:14:17,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1a871e3e8d67403a9f4dc6b7ac44ea18 is 50, key is test_row_0/B:col10/1733397257137/Put/seqid=0 2024-12-05T11:14:17,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741877_1053 (size=12151) 2024-12-05T11:14:17,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397317815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1a871e3e8d67403a9f4dc6b7ac44ea18 2024-12-05T11:14:17,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397317815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397317817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:17,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/35e091a2a66d4af2b90169bd13d93e8d is 50, key is test_row_0/C:col10/1733397257137/Put/seqid=0 2024-12-05T11:14:17,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741878_1054 (size=12151) 2024-12-05T11:14:17,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/35e091a2a66d4af2b90169bd13d93e8d 2024-12-05T11:14:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/222c1d0b81a6404c9b5f14b9abc5f79b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/222c1d0b81a6404c9b5f14b9abc5f79b 2024-12-05T11:14:17,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/222c1d0b81a6404c9b5f14b9abc5f79b, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T11:14:17,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1a871e3e8d67403a9f4dc6b7ac44ea18 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1a871e3e8d67403a9f4dc6b7ac44ea18 2024-12-05T11:14:17,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1a871e3e8d67403a9f4dc6b7ac44ea18, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T11:14:17,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/35e091a2a66d4af2b90169bd13d93e8d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/35e091a2a66d4af2b90169bd13d93e8d 2024-12-05T11:14:17,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/35e091a2a66d4af2b90169bd13d93e8d, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T11:14:17,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 152ms, sequenceid=251, compaction requested=true 2024-12-05T11:14:17,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:17,924 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:17,924 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:17,925 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:17,925 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:17,925 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:17,925 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/bd516ed0bc434089a6d5a435a29764e3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/62940dad079944818cbf8a18cc05030d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/222c1d0b81a6404c9b5f14b9abc5f79b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=38.3 K 2024-12-05T11:14:17,926 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:17,926 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:17,927 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:17,927 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b5012ec1881440f6b005d9d0dbf73cee, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/18023e32b3934313b0999eff0345d3f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1a871e3e8d67403a9f4dc6b7ac44ea18] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=36.0 K 2024-12-05T11:14:17,927 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd516ed0bc434089a6d5a435a29764e3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733397255257 2024-12-05T11:14:17,927 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62940dad079944818cbf8a18cc05030d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733397256440 2024-12-05T11:14:17,927 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b5012ec1881440f6b005d9d0dbf73cee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733397255257 2024-12-05T11:14:17,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397257803, max timestamp=1733397257929Num uniques 859; 2024-12-05T11:14:17,933 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 18023e32b3934313b0999eff0345d3f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733397256440 2024-12-05T11:14:17,933 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222c1d0b81a6404c9b5f14b9abc5f79b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733397257131 2024-12-05T11:14:17,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397257803, max timestamp=1733397257929Num uniques 859; 2024-12-05T11:14:17,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:17,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:17,934 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a871e3e8d67403a9f4dc6b7ac44ea18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733397257131 2024-12-05T11:14:17,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-05T11:14:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:17,952 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:17,953 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/c674526ae2b24a5c919da6f7533ceab0 is 50, key is test_row_0/A:col10/1733397257137/Put/seqid=0 2024-12-05T11:14:17,956 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:17,958 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3db4b9b6d24b457a850d4ccfd8316639 is 50, key is test_row_0/B:col10/1733397257137/Put/seqid=0 2024-12-05T11:14:17,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/a1d5f562319448ba9b1582efc3a25706 is 50, key is test_row_0/A:col10/1733397257936/Put/seqid=0 2024-12-05T11:14:17,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741879_1055 (size=12629) 2024-12-05T11:14:17,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741880_1056 (size=12629) 2024-12-05T11:14:17,973 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/c674526ae2b24a5c919da6f7533ceab0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c674526ae2b24a5c919da6f7533ceab0 2024-12-05T11:14:17,980 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3db4b9b6d24b457a850d4ccfd8316639 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3db4b9b6d24b457a850d4ccfd8316639 2024-12-05T11:14:18,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741881_1057 (size=17181) 2024-12-05T11:14:18,006 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 3db4b9b6d24b457a850d4ccfd8316639(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:18,006 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into c674526ae2b24a5c919da6f7533ceab0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:18,006 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:18,006 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:18,006 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=13, startTime=1733397257923; duration=0sec 2024-12-05T11:14:18,006 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=13, startTime=1733397257924; duration=0sec 2024-12-05T11:14:18,007 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:18,007 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:18,007 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:18,007 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:18,007 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:18,009 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:18,009 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:18,009 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,010 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/4613964f5e8c42129e5cf8d096496741, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/23b72e11e17b4d5faa9d9ee766d1aa9c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/35e091a2a66d4af2b90169bd13d93e8d] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=36.0 K 2024-12-05T11:14:18,011 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 4613964f5e8c42129e5cf8d096496741, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733397255257 2024-12-05T11:14:18,011 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b72e11e17b4d5faa9d9ee766d1aa9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733397256440 2024-12-05T11:14:18,012 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e091a2a66d4af2b90169bd13d93e8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733397257131 2024-12-05T11:14:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397318009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397318019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397318020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,031 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:18,031 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/551ac8e8fb2642348f01a93da1787562 is 50, key is test_row_0/C:col10/1733397257137/Put/seqid=0 2024-12-05T11:14:18,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741882_1058 (size=12629) 2024-12-05T11:14:18,066 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/551ac8e8fb2642348f01a93da1787562 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/551ac8e8fb2642348f01a93da1787562 2024-12-05T11:14:18,076 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 551ac8e8fb2642348f01a93da1787562(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:18,076 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:18,076 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=13, startTime=1733397257924; duration=0sec 2024-12-05T11:14:18,076 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:18,077 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T11:14:18,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:18,130 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-05T11:14:18,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397318129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397318129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397318129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,137 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:18,141 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:18,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:18,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397318338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397318339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397318339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/a1d5f562319448ba9b1582efc3a25706 2024-12-05T11:14:18,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/0f0d68035f214ea787ee5d09579ed85c is 50, key is test_row_0/B:col10/1733397257936/Put/seqid=0 2024-12-05T11:14:18,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741883_1059 (size=12301) 2024-12-05T11:14:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:18,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:18,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397318648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397318648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397318648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:18,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/0f0d68035f214ea787ee5d09579ed85c 2024-12-05T11:14:18,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/adf0b1d0e028444e8f5d430cee33cc60 is 50, key is test_row_0/C:col10/1733397257936/Put/seqid=0 2024-12-05T11:14:18,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741884_1060 (size=12301) 2024-12-05T11:14:18,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:18,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:18,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:18,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:19,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:19,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:19,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:19,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397319159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:19,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397319162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:19,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397319163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:19,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:19,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:19,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/adf0b1d0e028444e8f5d430cee33cc60 2024-12-05T11:14:19,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/a1d5f562319448ba9b1582efc3a25706 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/a1d5f562319448ba9b1582efc3a25706 2024-12-05T11:14:19,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/a1d5f562319448ba9b1582efc3a25706, entries=250, sequenceid=273, filesize=16.8 K 2024-12-05T11:14:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/0f0d68035f214ea787ee5d09579ed85c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/0f0d68035f214ea787ee5d09579ed85c 2024-12-05T11:14:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:19,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/0f0d68035f214ea787ee5d09579ed85c, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T11:14:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/adf0b1d0e028444e8f5d430cee33cc60 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/adf0b1d0e028444e8f5d430cee33cc60 2024-12-05T11:14:19,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/adf0b1d0e028444e8f5d430cee33cc60, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T11:14:19,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1344ms, sequenceid=273, compaction requested=false 2024-12-05T11:14:19,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:19,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T11:14:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:19,384 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:14:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/38230db9681b40989cc3106299475eb6 is 50, key is test_row_0/A:col10/1733397258006/Put/seqid=0 2024-12-05T11:14:19,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741885_1061 (size=12301) 2024-12-05T11:14:19,798 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/38230db9681b40989cc3106299475eb6 2024-12-05T11:14:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/d173674a91cb4bd0a35ed42f7d81c9f7 is 50, key is test_row_0/B:col10/1733397258006/Put/seqid=0 2024-12-05T11:14:19,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741886_1062 (size=12301) 2024-12-05T11:14:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397320189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397320189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397320196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/d173674a91cb4bd0a35ed42f7d81c9f7 2024-12-05T11:14:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/126feeb9dce5481fa0b4f789cac77220 is 50, key is test_row_0/C:col10/1733397258006/Put/seqid=0 2024-12-05T11:14:20,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741887_1063 (size=12301) 2024-12-05T11:14:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:20,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397320308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397320308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397320309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397320518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397320518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397320518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,631 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/126feeb9dce5481fa0b4f789cac77220 2024-12-05T11:14:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/38230db9681b40989cc3106299475eb6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/38230db9681b40989cc3106299475eb6 2024-12-05T11:14:20,669 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/38230db9681b40989cc3106299475eb6, entries=150, sequenceid=296, filesize=12.0 K 2024-12-05T11:14:20,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/d173674a91cb4bd0a35ed42f7d81c9f7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d173674a91cb4bd0a35ed42f7d81c9f7 2024-12-05T11:14:20,677 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d173674a91cb4bd0a35ed42f7d81c9f7, entries=150, sequenceid=296, filesize=12.0 K 2024-12-05T11:14:20,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/126feeb9dce5481fa0b4f789cac77220 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/126feeb9dce5481fa0b4f789cac77220 2024-12-05T11:14:20,683 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/126feeb9dce5481fa0b4f789cac77220, entries=150, sequenceid=296, filesize=12.0 K 2024-12-05T11:14:20,684 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1300ms, sequenceid=296, compaction requested=true 2024-12-05T11:14:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T11:14:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-05T11:14:20,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T11:14:20,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5450 sec 2024-12-05T11:14:20,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.5600 sec 2024-12-05T11:14:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:20,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:14:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:20,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/5fcacfbc5d964b9da4ba88f421d59de3 is 50, key is test_row_0/A:col10/1733397260833/Put/seqid=0 2024-12-05T11:14:20,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741888_1064 (size=12301) 2024-12-05T11:14:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397320858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397320861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397320862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397320968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397320968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397320968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397321178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397321178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397321178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/5fcacfbc5d964b9da4ba88f421d59de3 2024-12-05T11:14:21,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/7f1ea138bb2a4846a2de57537d47c071 is 50, key is test_row_0/B:col10/1733397260833/Put/seqid=0 2024-12-05T11:14:21,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741889_1065 (size=12301) 2024-12-05T11:14:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397321488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397321488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397321488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/7f1ea138bb2a4846a2de57537d47c071 2024-12-05T11:14:21,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37672 deadline: 1733397321699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/b1a408392d80473a8ca199616798bf4e is 50, key is test_row_0/C:col10/1733397260833/Put/seqid=0 2024-12-05T11:14:21,704 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37676 deadline: 1733397321729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:21,735 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:21,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741890_1066 (size=12301) 2024-12-05T11:14:21,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/b1a408392d80473a8ca199616798bf4e 2024-12-05T11:14:21,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/5fcacfbc5d964b9da4ba88f421d59de3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5fcacfbc5d964b9da4ba88f421d59de3 2024-12-05T11:14:21,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5fcacfbc5d964b9da4ba88f421d59de3, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T11:14:21,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/7f1ea138bb2a4846a2de57537d47c071 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7f1ea138bb2a4846a2de57537d47c071 2024-12-05T11:14:21,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7f1ea138bb2a4846a2de57537d47c071, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T11:14:21,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/b1a408392d80473a8ca199616798bf4e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/b1a408392d80473a8ca199616798bf4e 2024-12-05T11:14:21,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/b1a408392d80473a8ca199616798bf4e, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T11:14:21,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 964ms, sequenceid=311, compaction requested=true 2024-12-05T11:14:21,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:21,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:21,798 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:21,798 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:21,799 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:21,799 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54412 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:21,799 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:21,799 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:21,799 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:21,799 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:21,800 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c674526ae2b24a5c919da6f7533ceab0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/a1d5f562319448ba9b1582efc3a25706, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/38230db9681b40989cc3106299475eb6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5fcacfbc5d964b9da4ba88f421d59de3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=53.1 K 2024-12-05T11:14:21,800 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3db4b9b6d24b457a850d4ccfd8316639, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/0f0d68035f214ea787ee5d09579ed85c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d173674a91cb4bd0a35ed42f7d81c9f7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7f1ea138bb2a4846a2de57537d47c071] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=48.4 K 2024-12-05T11:14:21,800 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db4b9b6d24b457a850d4ccfd8316639, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733397257131 2024-12-05T11:14:21,801 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c674526ae2b24a5c919da6f7533ceab0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733397257131 2024-12-05T11:14:21,801 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f0d68035f214ea787ee5d09579ed85c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733397257803 2024-12-05T11:14:21,801 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d173674a91cb4bd0a35ed42f7d81c9f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733397258006 2024-12-05T11:14:21,801 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d5f562319448ba9b1582efc3a25706, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733397257803 2024-12-05T11:14:21,802 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f1ea138bb2a4846a2de57537d47c071, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733397260831 2024-12-05T11:14:21,802 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38230db9681b40989cc3106299475eb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733397258006 2024-12-05T11:14:21,802 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fcacfbc5d964b9da4ba88f421d59de3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733397260831 2024-12-05T11:14:21,819 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:21,820 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/90a83d697ce8446ebcb787cf07089ced is 50, key is test_row_0/A:col10/1733397260833/Put/seqid=0 2024-12-05T11:14:21,820 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:21,821 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/bf3c4570ef684e06a4fb71d77eb2848d is 50, key is test_row_0/B:col10/1733397260833/Put/seqid=0 2024-12-05T11:14:21,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741891_1067 (size=12915) 2024-12-05T11:14:21,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741892_1068 (size=12915) 2024-12-05T11:14:21,893 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/90a83d697ce8446ebcb787cf07089ced as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/90a83d697ce8446ebcb787cf07089ced 2024-12-05T11:14:21,895 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/bf3c4570ef684e06a4fb71d77eb2848d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/bf3c4570ef684e06a4fb71d77eb2848d 2024-12-05T11:14:21,902 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 90a83d697ce8446ebcb787cf07089ced(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:21,902 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:21,902 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=12, startTime=1733397261797; duration=0sec 2024-12-05T11:14:21,902 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:21,903 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:21,903 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:21,905 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:21,905 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:21,905 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:21,906 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/551ac8e8fb2642348f01a93da1787562, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/adf0b1d0e028444e8f5d430cee33cc60, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/126feeb9dce5481fa0b4f789cac77220, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/b1a408392d80473a8ca199616798bf4e] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=48.4 K 2024-12-05T11:14:21,906 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into bf3c4570ef684e06a4fb71d77eb2848d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:21,906 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:21,906 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=12, startTime=1733397261798; duration=0sec 2024-12-05T11:14:21,906 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:21,906 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551ac8e8fb2642348f01a93da1787562, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733397257131 2024-12-05T11:14:21,906 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:21,906 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf0b1d0e028444e8f5d430cee33cc60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733397257803 2024-12-05T11:14:21,907 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 126feeb9dce5481fa0b4f789cac77220, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1733397258006 2024-12-05T11:14:21,907 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1a408392d80473a8ca199616798bf4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733397260831 2024-12-05T11:14:21,925 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:21,925 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ac987818eca041e1a812855d34bfec35 is 50, key is test_row_0/C:col10/1733397260833/Put/seqid=0 2024-12-05T11:14:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741893_1069 (size=12915) 2024-12-05T11:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:22,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:14:22,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:22,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/c9321af5d4cb40a6a21ad99fbf30f457 is 50, key is test_row_0/A:col10/1733397261999/Put/seqid=0 2024-12-05T11:14:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741894_1070 (size=12301) 2024-12-05T11:14:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397322024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397322030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397322033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397322138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397322139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397322149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T11:14:22,299 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:22,299 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-05T11:14:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:14:22,303 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:22,304 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:22,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:22,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397322348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397322349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,352 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ac987818eca041e1a812855d34bfec35 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac987818eca041e1a812855d34bfec35 2024-12-05T11:14:22,359 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into ac987818eca041e1a812855d34bfec35(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:22,359 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:22,359 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=12, startTime=1733397261798; duration=0sec 2024-12-05T11:14:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397322359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,360 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:22,360 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:14:22,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/c9321af5d4cb40a6a21ad99fbf30f457 2024-12-05T11:14:22,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/956064382c204511868777e9aad93fe1 is 50, key is test_row_0/B:col10/1733397261999/Put/seqid=0 2024-12-05T11:14:22,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741895_1071 (size=12301) 2024-12-05T11:14:22,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/956064382c204511868777e9aad93fe1 2024-12-05T11:14:22,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:14:22,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:22,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:22,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6329d469e30b4d25a2e485756e80516e is 50, key is test_row_0/C:col10/1733397261999/Put/seqid=0 2024-12-05T11:14:22,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741896_1072 (size=12301) 2024-12-05T11:14:22,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6329d469e30b4d25a2e485756e80516e 2024-12-05T11:14:22,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/c9321af5d4cb40a6a21ad99fbf30f457 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c9321af5d4cb40a6a21ad99fbf30f457 2024-12-05T11:14:22,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c9321af5d4cb40a6a21ad99fbf30f457, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T11:14:22,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/956064382c204511868777e9aad93fe1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/956064382c204511868777e9aad93fe1 2024-12-05T11:14:22,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/956064382c204511868777e9aad93fe1, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T11:14:22,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6329d469e30b4d25a2e485756e80516e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6329d469e30b4d25a2e485756e80516e 2024-12-05T11:14:22,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6329d469e30b4d25a2e485756e80516e, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T11:14:22,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 506ms, sequenceid=332, compaction requested=false 2024-12-05T11:14:22,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:22,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T11:14:22,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:22,612 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:14:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:22,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:22,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/f65083079a1a466a9515a0a334bba59e is 50, key is test_row_0/A:col10/1733397262028/Put/seqid=0 2024-12-05T11:14:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:14:22,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741897_1073 (size=12301) 2024-12-05T11:14:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:22,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397322692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397322695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397322695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397322798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397322808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397322809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:14:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397323008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397323018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397323019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,035 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/f65083079a1a466a9515a0a334bba59e 2024-12-05T11:14:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/5cbac49fdceb4a7dae1a41ba05466071 is 50, key is test_row_0/B:col10/1733397262028/Put/seqid=0 2024-12-05T11:14:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741898_1074 (size=12301) 2024-12-05T11:14:23,055 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/5cbac49fdceb4a7dae1a41ba05466071 2024-12-05T11:14:23,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ba57e0d2def14b338e8332465d01ed76 is 50, key is test_row_0/C:col10/1733397262028/Put/seqid=0 2024-12-05T11:14:23,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741899_1075 (size=12301) 2024-12-05T11:14:23,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397323318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397323328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397323329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:14:23,477 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ba57e0d2def14b338e8332465d01ed76 2024-12-05T11:14:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/f65083079a1a466a9515a0a334bba59e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/f65083079a1a466a9515a0a334bba59e 2024-12-05T11:14:23,493 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/f65083079a1a466a9515a0a334bba59e, entries=150, sequenceid=347, filesize=12.0 K 2024-12-05T11:14:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/5cbac49fdceb4a7dae1a41ba05466071 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/5cbac49fdceb4a7dae1a41ba05466071 2024-12-05T11:14:23,501 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/5cbac49fdceb4a7dae1a41ba05466071, entries=150, sequenceid=347, filesize=12.0 K 2024-12-05T11:14:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ba57e0d2def14b338e8332465d01ed76 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ba57e0d2def14b338e8332465d01ed76 2024-12-05T11:14:23,509 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ba57e0d2def14b338e8332465d01ed76, entries=150, sequenceid=347, filesize=12.0 K 2024-12-05T11:14:23,510 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 898ms, sequenceid=347, compaction requested=true 2024-12-05T11:14:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T11:14:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-05T11:14:23,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T11:14:23,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2080 sec 2024-12-05T11:14:23,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2190 sec 2024-12-05T11:14:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:23,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:14:23,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:23,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:23,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:23,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:23,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:23,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:23,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/d196d90cc65d4c7092e4ac6acd1cb62e is 50, key is test_row_0/A:col10/1733397262690/Put/seqid=0 2024-12-05T11:14:23,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741900_1076 (size=14741) 2024-12-05T11:14:23,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397323851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397323854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397323856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397323959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397323968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397323968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397324168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397324178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397324178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/d196d90cc65d4c7092e4ac6acd1cb62e 2024-12-05T11:14:24,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3f2ff4356033470ab41b34e53db2cafc is 50, key is test_row_0/B:col10/1733397262690/Put/seqid=0 2024-12-05T11:14:24,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741901_1077 (size=12301) 2024-12-05T11:14:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T11:14:24,449 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:24,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-05T11:14:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:14:24,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:24,452 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:24,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:24,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397324479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397324488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397324488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:14:24,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:14:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:24,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3f2ff4356033470ab41b34e53db2cafc 2024-12-05T11:14:24,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9b91442ef9fe4fbbbada8c5d42d03196 is 50, key is test_row_0/C:col10/1733397262690/Put/seqid=0 2024-12-05T11:14:24,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741902_1078 (size=12301) 2024-12-05T11:14:24,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:14:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:24,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:14:24,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:14:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:24,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:24,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397324988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397324998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:25,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397324999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:25,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:25,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:25,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:25,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9b91442ef9fe4fbbbada8c5d42d03196 2024-12-05T11:14:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:14:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/d196d90cc65d4c7092e4ac6acd1cb62e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d196d90cc65d4c7092e4ac6acd1cb62e 2024-12-05T11:14:25,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d196d90cc65d4c7092e4ac6acd1cb62e, entries=200, sequenceid=367, filesize=14.4 K 2024-12-05T11:14:25,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3f2ff4356033470ab41b34e53db2cafc as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3f2ff4356033470ab41b34e53db2cafc 2024-12-05T11:14:25,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3f2ff4356033470ab41b34e53db2cafc, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T11:14:25,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9b91442ef9fe4fbbbada8c5d42d03196 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9b91442ef9fe4fbbbada8c5d42d03196 2024-12-05T11:14:25,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9b91442ef9fe4fbbbada8c5d42d03196, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T11:14:25,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1276ms, sequenceid=367, compaction requested=true 2024-12-05T11:14:25,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:25,109 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:25,109 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:25,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:25,110 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:25,110 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:25,110 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:25,110 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/90a83d697ce8446ebcb787cf07089ced, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c9321af5d4cb40a6a21ad99fbf30f457, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/f65083079a1a466a9515a0a334bba59e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d196d90cc65d4c7092e4ac6acd1cb62e] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=51.0 K 2024-12-05T11:14:25,111 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:25,111 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:25,111 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90a83d697ce8446ebcb787cf07089ced, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733397260831 2024-12-05T11:14:25,111 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:25,111 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/bf3c4570ef684e06a4fb71d77eb2848d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/956064382c204511868777e9aad93fe1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/5cbac49fdceb4a7dae1a41ba05466071, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3f2ff4356033470ab41b34e53db2cafc] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=48.7 K 2024-12-05T11:14:25,111 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9321af5d4cb40a6a21ad99fbf30f457, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733397260848 2024-12-05T11:14:25,111 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3c4570ef684e06a4fb71d77eb2848d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733397260831 2024-12-05T11:14:25,112 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65083079a1a466a9515a0a334bba59e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733397262020 2024-12-05T11:14:25,112 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 956064382c204511868777e9aad93fe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733397260848 2024-12-05T11:14:25,112 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d196d90cc65d4c7092e4ac6acd1cb62e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733397262690 2024-12-05T11:14:25,113 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbac49fdceb4a7dae1a41ba05466071, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733397262020 2024-12-05T11:14:25,113 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f2ff4356033470ab41b34e53db2cafc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733397262690 2024-12-05T11:14:25,132 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:25,133 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/e052dcba41b64344900baff4e516b29c is 50, key is test_row_0/A:col10/1733397262690/Put/seqid=0 2024-12-05T11:14:25,139 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:25,140 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/ae461f584fb54e46ac66f904fb48bfba is 50, key is test_row_0/B:col10/1733397262690/Put/seqid=0 2024-12-05T11:14:25,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741903_1079 (size=13051) 2024-12-05T11:14:25,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741904_1080 (size=13051) 2024-12-05T11:14:25,155 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/e052dcba41b64344900baff4e516b29c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/e052dcba41b64344900baff4e516b29c 2024-12-05T11:14:25,160 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/ae461f584fb54e46ac66f904fb48bfba as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/ae461f584fb54e46ac66f904fb48bfba 2024-12-05T11:14:25,164 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into e052dcba41b64344900baff4e516b29c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:25,164 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:25,165 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=12, startTime=1733397265108; duration=0sec 2024-12-05T11:14:25,165 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:25,165 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:25,166 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:25,168 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into ae461f584fb54e46ac66f904fb48bfba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:25,168 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:25,168 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=12, startTime=1733397265109; duration=0sec 2024-12-05T11:14:25,168 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:25,168 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:25,169 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:25,169 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:25,169 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:25,169 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac987818eca041e1a812855d34bfec35, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6329d469e30b4d25a2e485756e80516e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ba57e0d2def14b338e8332465d01ed76, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9b91442ef9fe4fbbbada8c5d42d03196] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=48.7 K 2024-12-05T11:14:25,169 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac987818eca041e1a812855d34bfec35, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733397260831 2024-12-05T11:14:25,170 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6329d469e30b4d25a2e485756e80516e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733397260848 2024-12-05T11:14:25,171 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba57e0d2def14b338e8332465d01ed76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733397262020 2024-12-05T11:14:25,171 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b91442ef9fe4fbbbada8c5d42d03196, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733397262690 2024-12-05T11:14:25,183 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:25,183 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/faaf72dbd7b14a27a857a72ca3fe0474 is 50, key is test_row_0/C:col10/1733397262690/Put/seqid=0 2024-12-05T11:14:25,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741905_1081 (size=13051) 2024-12-05T11:14:25,205 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/faaf72dbd7b14a27a857a72ca3fe0474 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/faaf72dbd7b14a27a857a72ca3fe0474 2024-12-05T11:14:25,214 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into faaf72dbd7b14a27a857a72ca3fe0474(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:25,214 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:25,214 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=12, startTime=1733397265109; duration=0sec 2024-12-05T11:14:25,214 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:25,214 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:25,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T11:14:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:25,219 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T11:14:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:25,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7a262d9d1da54317a8103ed3d6ba17d5 is 50, key is test_row_0/A:col10/1733397263849/Put/seqid=0 2024-12-05T11:14:25,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741906_1082 (size=12301) 2024-12-05T11:14:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:14:25,634 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7a262d9d1da54317a8103ed3d6ba17d5 2024-12-05T11:14:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3a908e757195423fbb3971ca9ac99b0d is 50, key is test_row_0/B:col10/1733397263849/Put/seqid=0 2024-12-05T11:14:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741907_1083 (size=12301) 2024-12-05T11:14:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397326037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397326039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397326038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,047 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3a908e757195423fbb3971ca9ac99b0d 2024-12-05T11:14:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6b59113a08ce46d68d57ad5acf92b261 is 50, key is test_row_0/C:col10/1733397263849/Put/seqid=0 2024-12-05T11:14:26,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741908_1084 (size=12301) 2024-12-05T11:14:26,066 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6b59113a08ce46d68d57ad5acf92b261 2024-12-05T11:14:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/7a262d9d1da54317a8103ed3d6ba17d5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a262d9d1da54317a8103ed3d6ba17d5 2024-12-05T11:14:26,080 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a262d9d1da54317a8103ed3d6ba17d5, entries=150, sequenceid=382, filesize=12.0 K 2024-12-05T11:14:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/3a908e757195423fbb3971ca9ac99b0d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3a908e757195423fbb3971ca9ac99b0d 2024-12-05T11:14:26,090 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3a908e757195423fbb3971ca9ac99b0d, entries=150, sequenceid=382, filesize=12.0 K 2024-12-05T11:14:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/6b59113a08ce46d68d57ad5acf92b261 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6b59113a08ce46d68d57ad5acf92b261 2024-12-05T11:14:26,098 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6b59113a08ce46d68d57ad5acf92b261, entries=150, sequenceid=382, filesize=12.0 K 2024-12-05T11:14:26,099 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 881ms, sequenceid=382, compaction requested=false 2024-12-05T11:14:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T11:14:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-05T11:14:26,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T11:14:26,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-12-05T11:14:26,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6570 sec 2024-12-05T11:14:26,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397266034, max timestamp=1733397266150Num uniques 859; 2024-12-05T11:14:26,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:26,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397266034, max timestamp=1733397266150Num uniques 859; 2024-12-05T11:14:26,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397266034, max timestamp=1733397266150Num uniques 859; 2024-12-05T11:14:26,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:26,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:26,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-12-05T11:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:26,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:26,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:26,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/9dac14d7680d45a7a2e9e38ef34abbc7 is 50, key is test_row_0/A:col10/1733397266156/Put/seqid=0 2024-12-05T11:14:26,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741909_1085 (size=14741) 2024-12-05T11:14:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397326196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397266189, max timestamp=1733397266201Num uniques 859; 2024-12-05T11:14:26,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397266189, max timestamp=1733397266201Num uniques 859; 2024-12-05T11:14:26,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:14:26,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:14:26,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397266189, max timestamp=1733397266201Num uniques 859; 2024-12-05T11:14:26,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:14:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397326249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397326251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397326308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397326359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397326368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397326518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397326569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/9dac14d7680d45a7a2e9e38ef34abbc7 2024-12-05T11:14:26,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397326578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/aedaf657e10f4778a72556245050e769 is 50, key is test_row_0/B:col10/1733397266156/Put/seqid=0 2024-12-05T11:14:26,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741910_1086 (size=12301) 2024-12-05T11:14:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T11:14:26,599 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:26,599 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-05T11:14:26,602 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:26,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:26,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:14:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:14:26,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:14:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:26,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:26,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397326828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397326878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397326899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:26,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:14:26,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:26,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:26,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:26,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:14:26,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/aedaf657e10f4778a72556245050e769 2024-12-05T11:14:27,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/f17176b7e6db419aadb10d4c0bfae24d is 50, key is test_row_0/C:col10/1733397266156/Put/seqid=0 2024-12-05T11:14:27,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741911_1087 (size=12301) 2024-12-05T11:14:27,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/f17176b7e6db419aadb10d4c0bfae24d 2024-12-05T11:14:27,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/9dac14d7680d45a7a2e9e38ef34abbc7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/9dac14d7680d45a7a2e9e38ef34abbc7 2024-12-05T11:14:27,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/9dac14d7680d45a7a2e9e38ef34abbc7, entries=200, sequenceid=408, filesize=14.4 K 2024-12-05T11:14:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/aedaf657e10f4778a72556245050e769 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/aedaf657e10f4778a72556245050e769 2024-12-05T11:14:27,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/aedaf657e10f4778a72556245050e769, entries=150, sequenceid=408, filesize=12.0 K 2024-12-05T11:14:27,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/f17176b7e6db419aadb10d4c0bfae24d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f17176b7e6db419aadb10d4c0bfae24d 2024-12-05T11:14:27,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f17176b7e6db419aadb10d4c0bfae24d, entries=150, sequenceid=408, filesize=12.0 K 2024-12-05T11:14:27,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=134.18 KB/137400 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 881ms, sequenceid=408, compaction requested=true 2024-12-05T11:14:27,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:27,040 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:27,040 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:27,041 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:27,041 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:27,041 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,042 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/ae461f584fb54e46ac66f904fb48bfba, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3a908e757195423fbb3971ca9ac99b0d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/aedaf657e10f4778a72556245050e769] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=36.8 K 2024-12-05T11:14:27,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:27,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:27,042 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,042 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/e052dcba41b64344900baff4e516b29c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a262d9d1da54317a8103ed3d6ba17d5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/9dac14d7680d45a7a2e9e38ef34abbc7] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=39.2 K 2024-12-05T11:14:27,043 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e052dcba41b64344900baff4e516b29c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733397262690 2024-12-05T11:14:27,043 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ae461f584fb54e46ac66f904fb48bfba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733397262690 2024-12-05T11:14:27,043 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a908e757195423fbb3971ca9ac99b0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733397263841 2024-12-05T11:14:27,044 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a262d9d1da54317a8103ed3d6ba17d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733397263841 2024-12-05T11:14:27,044 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting aedaf657e10f4778a72556245050e769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733397266152 2024-12-05T11:14:27,044 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dac14d7680d45a7a2e9e38ef34abbc7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733397266152 2024-12-05T11:14:27,052 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:27,052 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/cd78f3e0dada45e4b2003795b0f6a560 is 50, key is test_row_0/B:col10/1733397266156/Put/seqid=0 2024-12-05T11:14:27,052 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:27,053 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/23ee945d6e504e9e9e10e27ae0d5b1dd is 50, key is test_row_0/A:col10/1733397266156/Put/seqid=0 2024-12-05T11:14:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741913_1089 (size=13153) 2024-12-05T11:14:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741912_1088 (size=13153) 2024-12-05T11:14:27,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T11:14:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,106 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T11:14:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:27,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/1d928abfc82a4aebba2045adc965d3cf is 50, key is test_row_0/A:col10/1733397266242/Put/seqid=0 2024-12-05T11:14:27,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741914_1090 (size=12301) 2024-12-05T11:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397327367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397327388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:27,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397327408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:27,468 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/23ee945d6e504e9e9e10e27ae0d5b1dd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/23ee945d6e504e9e9e10e27ae0d5b1dd 2024-12-05T11:14:27,468 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/cd78f3e0dada45e4b2003795b0f6a560 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/cd78f3e0dada45e4b2003795b0f6a560 2024-12-05T11:14:27,476 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 23ee945d6e504e9e9e10e27ae0d5b1dd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:27,476 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into cd78f3e0dada45e4b2003795b0f6a560(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:27,477 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=13, startTime=1733397267040; duration=0sec 2024-12-05T11:14:27,477 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=13, startTime=1733397267040; duration=0sec 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:27,477 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:27,478 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:27,478 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:27,478 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,478 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/faaf72dbd7b14a27a857a72ca3fe0474, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6b59113a08ce46d68d57ad5acf92b261, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f17176b7e6db419aadb10d4c0bfae24d] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=36.8 K 2024-12-05T11:14:27,479 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting faaf72dbd7b14a27a857a72ca3fe0474, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733397262690 2024-12-05T11:14:27,479 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b59113a08ce46d68d57ad5acf92b261, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733397263841 2024-12-05T11:14:27,480 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f17176b7e6db419aadb10d4c0bfae24d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733397266152 2024-12-05T11:14:27,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397327478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:27,488 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:27,488 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/cee952928ac846528044a7d1db770ad1 is 50, key is test_row_0/C:col10/1733397266156/Put/seqid=0 2024-12-05T11:14:27,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741915_1091 (size=13153) 2024-12-05T11:14:27,517 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/1d928abfc82a4aebba2045adc965d3cf 2024-12-05T11:14:27,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/a28a4369e7a54001b6696bdc7972c204 is 50, key is test_row_0/B:col10/1733397266242/Put/seqid=0 2024-12-05T11:14:27,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741916_1092 (size=12301) 2024-12-05T11:14:27,531 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/a28a4369e7a54001b6696bdc7972c204 2024-12-05T11:14:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/d1e937dabc4f45f6af740f01e0a5d984 is 50, key is test_row_0/C:col10/1733397266242/Put/seqid=0 2024-12-05T11:14:27,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741917_1093 (size=12301) 2024-12-05T11:14:27,544 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/d1e937dabc4f45f6af740f01e0a5d984 2024-12-05T11:14:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/1d928abfc82a4aebba2045adc965d3cf as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/1d928abfc82a4aebba2045adc965d3cf 2024-12-05T11:14:27,557 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/1d928abfc82a4aebba2045adc965d3cf, entries=150, sequenceid=444, filesize=12.0 K 2024-12-05T11:14:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/a28a4369e7a54001b6696bdc7972c204 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/a28a4369e7a54001b6696bdc7972c204 2024-12-05T11:14:27,565 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/a28a4369e7a54001b6696bdc7972c204, entries=150, sequenceid=444, filesize=12.0 K 2024-12-05T11:14:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/d1e937dabc4f45f6af740f01e0a5d984 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/d1e937dabc4f45f6af740f01e0a5d984 2024-12-05T11:14:27,575 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/d1e937dabc4f45f6af740f01e0a5d984, entries=150, sequenceid=444, filesize=12.0 K 2024-12-05T11:14:27,577 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 471ms, sequenceid=444, compaction requested=false 2024-12-05T11:14:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T11:14:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-05T11:14:27,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T11:14:27,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 976 msec 2024-12-05T11:14:27,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 983 msec 2024-12-05T11:14:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:27,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:27,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/550441ebce934c19bc05eaf74321dc9f is 50, key is test_row_0/A:col10/1733397267688/Put/seqid=0 2024-12-05T11:14:27,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741918_1094 (size=14741) 2024-12-05T11:14:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T11:14:27,739 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:27,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-05T11:14:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:27,742 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:27,743 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:27,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:27,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397267709, max timestamp=1733397267757Num uniques 859; 2024-12-05T11:14:27,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397267709, max timestamp=1733397267757Num uniques 859; 2024-12-05T11:14:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397267709, max timestamp=1733397267757Num uniques 859; 2024-12-05T11:14:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:27,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397327821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:27,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:27,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:27,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:27,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:27,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:27,903 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/cee952928ac846528044a7d1db770ad1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/cee952928ac846528044a7d1db770ad1 2024-12-05T11:14:27,911 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into cee952928ac846528044a7d1db770ad1(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:27,912 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:27,912 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=13, startTime=1733397267040; duration=0sec 2024-12-05T11:14:27,912 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:27,912 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:27,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397327938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:28,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:28,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/550441ebce934c19bc05eaf74321dc9f 2024-12-05T11:14:28,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/b99ab81662c941e7b37bf5ace80963bd is 50, key is test_row_0/B:col10/1733397267688/Put/seqid=0 2024-12-05T11:14:28,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741919_1095 (size=12301) 2024-12-05T11:14:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397328149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:28,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:28,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397328409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:28,410 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:28,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397328428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:28,429 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:28,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397328458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:28,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/b99ab81662c941e7b37bf5ace80963bd 2024-12-05T11:14:28,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9710636d3b3e47f380468e11c82c8615 is 50, key is test_row_0/C:col10/1733397267688/Put/seqid=0 2024-12-05T11:14:28,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741920_1096 (size=12301) 2024-12-05T11:14:28,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:28,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9710636d3b3e47f380468e11c82c8615 2024-12-05T11:14:28,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/550441ebce934c19bc05eaf74321dc9f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/550441ebce934c19bc05eaf74321dc9f 2024-12-05T11:14:28,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/550441ebce934c19bc05eaf74321dc9f, entries=200, sequenceid=457, filesize=14.4 K 2024-12-05T11:14:28,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/b99ab81662c941e7b37bf5ace80963bd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b99ab81662c941e7b37bf5ace80963bd 2024-12-05T11:14:28,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b99ab81662c941e7b37bf5ace80963bd, entries=150, sequenceid=457, filesize=12.0 K 2024-12-05T11:14:28,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9710636d3b3e47f380468e11c82c8615 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9710636d3b3e47f380468e11c82c8615 2024-12-05T11:14:28,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9710636d3b3e47f380468e11c82c8615, entries=150, sequenceid=457, filesize=12.0 K 2024-12-05T11:14:28,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1267ms, sequenceid=457, compaction requested=true 2024-12-05T11:14:28,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:28,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:28,956 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:28,957 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:28,959 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:28,959 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:28,959 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:28,959 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:28,959 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,959 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,959 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/23ee945d6e504e9e9e10e27ae0d5b1dd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/1d928abfc82a4aebba2045adc965d3cf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/550441ebce934c19bc05eaf74321dc9f] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=39.3 K 2024-12-05T11:14:28,959 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/cd78f3e0dada45e4b2003795b0f6a560, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/a28a4369e7a54001b6696bdc7972c204, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b99ab81662c941e7b37bf5ace80963bd] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=36.9 K 2024-12-05T11:14:28,959 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ee945d6e504e9e9e10e27ae0d5b1dd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733397266152 2024-12-05T11:14:28,959 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting cd78f3e0dada45e4b2003795b0f6a560, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733397266152 2024-12-05T11:14:28,960 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d928abfc82a4aebba2045adc965d3cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733397266242 2024-12-05T11:14:28,960 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting a28a4369e7a54001b6696bdc7972c204, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733397266242 2024-12-05T11:14:28,960 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550441ebce934c19bc05eaf74321dc9f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733397267355 2024-12-05T11:14:28,960 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b99ab81662c941e7b37bf5ace80963bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733397267361 2024-12-05T11:14:28,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T11:14:28,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:28,968 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T11:14:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:28,970 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:28,970 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/be77d2b21459463d923633a966eb25a1 is 50, key is test_row_0/A:col10/1733397267688/Put/seqid=0 2024-12-05T11:14:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:28,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. as already flushing 2024-12-05T11:14:28,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741921_1097 (size=13255) 2024-12-05T11:14:28,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/8aa6469c40764aa69dd9312d8ba86330 is 50, key is test_row_0/A:col10/1733397267811/Put/seqid=0 2024-12-05T11:14:28,990 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:28,991 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/930a2160db7c413b9839734d7920b574 is 50, key is test_row_0/B:col10/1733397267688/Put/seqid=0 2024-12-05T11:14:29,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741923_1099 (size=13255) 2024-12-05T11:14:29,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741922_1098 (size=12301) 2024-12-05T11:14:29,010 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/930a2160db7c413b9839734d7920b574 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/930a2160db7c413b9839734d7920b574 2024-12-05T11:14:29,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397329010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:29,017 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 930a2160db7c413b9839734d7920b574(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:29,017 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:29,017 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=13, startTime=1733397268957; duration=0sec 2024-12-05T11:14:29,017 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:29,017 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:29,017 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:29,018 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:29,018 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:29,018 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:29,018 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/cee952928ac846528044a7d1db770ad1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/d1e937dabc4f45f6af740f01e0a5d984, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9710636d3b3e47f380468e11c82c8615] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=36.9 K 2024-12-05T11:14:29,019 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting cee952928ac846528044a7d1db770ad1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733397266152 2024-12-05T11:14:29,019 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e937dabc4f45f6af740f01e0a5d984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733397266242 2024-12-05T11:14:29,020 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 9710636d3b3e47f380468e11c82c8615, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733397267361 2024-12-05T11:14:29,031 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:29,032 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/7b0785a0fffb43b18fe2c4bf90611927 is 50, key is test_row_0/C:col10/1733397267688/Put/seqid=0 2024-12-05T11:14:29,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741924_1100 (size=13255) 2024-12-05T11:14:29,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397329118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:29,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397329328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:29,389 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/be77d2b21459463d923633a966eb25a1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/be77d2b21459463d923633a966eb25a1 2024-12-05T11:14:29,394 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into be77d2b21459463d923633a966eb25a1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:29,395 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:29,395 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=13, startTime=1733397268956; duration=0sec 2024-12-05T11:14:29,395 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:29,395 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:29,403 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/8aa6469c40764aa69dd9312d8ba86330 2024-12-05T11:14:29,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/4af3d5276b3a45e4a66ff343edcb68a2 is 50, key is test_row_0/B:col10/1733397267811/Put/seqid=0 2024-12-05T11:14:29,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741925_1101 (size=12301) 2024-12-05T11:14:29,448 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/7b0785a0fffb43b18fe2c4bf90611927 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/7b0785a0fffb43b18fe2c4bf90611927 2024-12-05T11:14:29,453 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 7b0785a0fffb43b18fe2c4bf90611927(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:29,453 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:29,453 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=13, startTime=1733397268957; duration=0sec 2024-12-05T11:14:29,454 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:29,454 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397329638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:29,817 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/4af3d5276b3a45e4a66ff343edcb68a2 2024-12-05T11:14:29,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/43eb9ae049024cd79ad99e1e07a7f5d0 is 50, key is test_row_0/C:col10/1733397267811/Put/seqid=0 2024-12-05T11:14:29,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741926_1102 (size=12301) 2024-12-05T11:14:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397330148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,230 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/43eb9ae049024cd79ad99e1e07a7f5d0 2024-12-05T11:14:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/8aa6469c40764aa69dd9312d8ba86330 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/8aa6469c40764aa69dd9312d8ba86330 2024-12-05T11:14:30,240 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/8aa6469c40764aa69dd9312d8ba86330, entries=150, sequenceid=494, filesize=12.0 K 2024-12-05T11:14:30,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/4af3d5276b3a45e4a66ff343edcb68a2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/4af3d5276b3a45e4a66ff343edcb68a2 2024-12-05T11:14:30,246 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/4af3d5276b3a45e4a66ff343edcb68a2, entries=150, sequenceid=494, filesize=12.0 K 2024-12-05T11:14:30,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/43eb9ae049024cd79ad99e1e07a7f5d0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/43eb9ae049024cd79ad99e1e07a7f5d0 2024-12-05T11:14:30,253 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/43eb9ae049024cd79ad99e1e07a7f5d0, entries=150, sequenceid=494, filesize=12.0 K 2024-12-05T11:14:30,255 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1286ms, sequenceid=494, compaction requested=false 2024-12-05T11:14:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-05T11:14:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-05T11:14:30,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T11:14:30,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5130 sec 2024-12-05T11:14:30,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.5220 sec 2024-12-05T11:14:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:30,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T11:14:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:30,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/fb71c328a3c448139bb70ba281162d52 is 50, key is test_row_0/A:col10/1733397269008/Put/seqid=0 2024-12-05T11:14:30,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741927_1103 (size=14741) 2024-12-05T11:14:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397330457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397330457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397330568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397330568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,643 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-05T11:14:30,643 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:30,645 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-05T11:14:30,645 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:30,645 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-05T11:14:30,645 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:30,646 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-05T11:14:30,646 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:30,648 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:30,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:30,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:30,648 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:30,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:30,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397330780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397330780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:30,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/fb71c328a3c448139bb70ba281162d52 2024-12-05T11:14:30,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/77b2db0672054ff8a5e4e803d5652a14 is 50, key is test_row_0/B:col10/1733397269008/Put/seqid=0 2024-12-05T11:14:30,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741928_1104 (size=12301) 2024-12-05T11:14:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37674 deadline: 1733397331090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37714 deadline: 1733397331090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37696 deadline: 1733397331158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:31,159 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in fb2f3b651a249ac1bc6c3ae631d8f7b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:14:31,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/77b2db0672054ff8a5e4e803d5652a14 2024-12-05T11:14:31,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ac7ae8f637f74a1da37edcc44afea906 is 50, key is test_row_0/C:col10/1733397269008/Put/seqid=0 2024-12-05T11:14:31,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741929_1105 (size=12301) 2024-12-05T11:14:31,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ac7ae8f637f74a1da37edcc44afea906 2024-12-05T11:14:31,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/fb71c328a3c448139bb70ba281162d52 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/fb71c328a3c448139bb70ba281162d52 2024-12-05T11:14:31,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/fb71c328a3c448139bb70ba281162d52, entries=200, sequenceid=509, filesize=14.4 K 2024-12-05T11:14:31,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/77b2db0672054ff8a5e4e803d5652a14 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/77b2db0672054ff8a5e4e803d5652a14 2024-12-05T11:14:31,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/77b2db0672054ff8a5e4e803d5652a14, entries=150, sequenceid=509, filesize=12.0 K 2024-12-05T11:14:31,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/ac7ae8f637f74a1da37edcc44afea906 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac7ae8f637f74a1da37edcc44afea906 2024-12-05T11:14:31,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac7ae8f637f74a1da37edcc44afea906, entries=150, sequenceid=509, filesize=12.0 K 2024-12-05T11:14:31,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 873ms, sequenceid=509, compaction requested=true 2024-12-05T11:14:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:31,293 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:31,293 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb2f3b651a249ac1bc6c3ae631d8f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:31,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:31,294 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:31,294 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:31,295 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/A is initiating minor compaction (all files) 2024-12-05T11:14:31,295 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/B is initiating minor compaction (all files) 2024-12-05T11:14:31,295 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/A in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:31,295 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/B in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:31,295 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/930a2160db7c413b9839734d7920b574, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/4af3d5276b3a45e4a66ff343edcb68a2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/77b2db0672054ff8a5e4e803d5652a14] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=37.0 K 2024-12-05T11:14:31,295 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/be77d2b21459463d923633a966eb25a1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/8aa6469c40764aa69dd9312d8ba86330, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/fb71c328a3c448139bb70ba281162d52] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=39.4 K 2024-12-05T11:14:31,295 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting be77d2b21459463d923633a966eb25a1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733397267361 2024-12-05T11:14:31,295 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930a2160db7c413b9839734d7920b574, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733397267361 2024-12-05T11:14:31,296 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa6469c40764aa69dd9312d8ba86330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1733397267811 2024-12-05T11:14:31,296 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af3d5276b3a45e4a66ff343edcb68a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1733397267811 2024-12-05T11:14:31,296 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting fb71c328a3c448139bb70ba281162d52, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733397268993 2024-12-05T11:14:31,296 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77b2db0672054ff8a5e4e803d5652a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733397269005 2024-12-05T11:14:31,303 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:31,303 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:31,304 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1857af97449240b4b948050bcffb0b0d is 50, key is test_row_0/B:col10/1733397269008/Put/seqid=0 2024-12-05T11:14:31,304 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/eecdc05123464d99a786a64b5b3d2986 is 50, key is test_row_0/A:col10/1733397269008/Put/seqid=0 2024-12-05T11:14:31,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741931_1107 (size=13357) 2024-12-05T11:14:31,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741930_1106 (size=13357) 2024-12-05T11:14:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:31,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-05T11:14:31,604 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-05T11:14:31,604 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:31,605 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,605 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-05T11:14:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:31,605 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:31,605 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:31,605 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:31,606 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:31,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:31,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:31,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/14bcf1517ec2468fb2f1cb01ea36baee is 50, key is test_row_0/A:col10/1733397271600/Put/seqid=0 2024-12-05T11:14:31,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741932_1108 (size=12301) 2024-12-05T11:14:31,720 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/eecdc05123464d99a786a64b5b3d2986 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/eecdc05123464d99a786a64b5b3d2986 2024-12-05T11:14:31,720 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1857af97449240b4b948050bcffb0b0d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1857af97449240b4b948050bcffb0b0d 2024-12-05T11:14:31,727 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/A of fb2f3b651a249ac1bc6c3ae631d8f7b9 into eecdc05123464d99a786a64b5b3d2986(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:31,727 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/B of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 1857af97449240b4b948050bcffb0b0d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:31,727 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:31,727 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:31,727 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/A, priority=13, startTime=1733397271293; duration=0sec 2024-12-05T11:14:31,728 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/B, priority=13, startTime=1733397271293; duration=0sec 2024-12-05T11:14:31,728 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:31,728 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:A 2024-12-05T11:14:31,728 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:31,728 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:31,728 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:B 2024-12-05T11:14:31,729 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:31,729 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): fb2f3b651a249ac1bc6c3ae631d8f7b9/C is initiating minor compaction (all files) 2024-12-05T11:14:31,729 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb2f3b651a249ac1bc6c3ae631d8f7b9/C in TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:31,729 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/7b0785a0fffb43b18fe2c4bf90611927, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/43eb9ae049024cd79ad99e1e07a7f5d0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac7ae8f637f74a1da37edcc44afea906] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp, totalSize=37.0 K 2024-12-05T11:14:31,730 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0785a0fffb43b18fe2c4bf90611927, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733397267361 2024-12-05T11:14:31,730 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 43eb9ae049024cd79ad99e1e07a7f5d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1733397267811 2024-12-05T11:14:31,731 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ac7ae8f637f74a1da37edcc44afea906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733397269005 2024-12-05T11:14:31,739 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb2f3b651a249ac1bc6c3ae631d8f7b9#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:31,739 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/22bc4f73fd724919bd178bbb5d484d09 is 50, key is test_row_0/C:col10/1733397269008/Put/seqid=0 2024-12-05T11:14:31,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741933_1109 (size=13357) 2024-12-05T11:14:31,750 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-05T11:14:31,750 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:31,750 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,750 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:31,831 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-05T11:14:31,832 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:31,832 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,832 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:31,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T11:14:31,910 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:32,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/14bcf1517ec2468fb2f1cb01ea36baee 2024-12-05T11:14:32,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1cb2276159ee4811a4a3b4cd0589e5c6 is 50, key is test_row_0/B:col10/1733397271600/Put/seqid=0 2024-12-05T11:14:32,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741934_1110 (size=12301) 2024-12-05T11:14:32,092 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:14:32,155 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/22bc4f73fd724919bd178bbb5d484d09 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/22bc4f73fd724919bd178bbb5d484d09 2024-12-05T11:14:32,164 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb2f3b651a249ac1bc6c3ae631d8f7b9/C of fb2f3b651a249ac1bc6c3ae631d8f7b9 into 22bc4f73fd724919bd178bbb5d484d09(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:32,164 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:32,164 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., storeName=fb2f3b651a249ac1bc6c3ae631d8f7b9/C, priority=13, startTime=1733397271293; duration=0sec 2024-12-05T11:14:32,165 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:32,165 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb2f3b651a249ac1bc6c3ae631d8f7b9:C 2024-12-05T11:14:32,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1cb2276159ee4811a4a3b4cd0589e5c6 2024-12-05T11:14:32,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/84aada0ea2d94d6c9df8ec0c6121c970 is 50, key is test_row_0/C:col10/1733397271600/Put/seqid=0 2024-12-05T11:14:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741935_1111 (size=12301) 2024-12-05T11:14:32,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/84aada0ea2d94d6c9df8ec0c6121c970 2024-12-05T11:14:32,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/14bcf1517ec2468fb2f1cb01ea36baee as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/14bcf1517ec2468fb2f1cb01ea36baee 2024-12-05T11:14:32,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/14bcf1517ec2468fb2f1cb01ea36baee, entries=150, sequenceid=531, filesize=12.0 K 2024-12-05T11:14:32,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/1cb2276159ee4811a4a3b4cd0589e5c6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1cb2276159ee4811a4a3b4cd0589e5c6 2024-12-05T11:14:32,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1cb2276159ee4811a4a3b4cd0589e5c6, entries=150, sequenceid=531, filesize=12.0 K 2024-12-05T11:14:32,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/84aada0ea2d94d6c9df8ec0c6121c970 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/84aada0ea2d94d6c9df8ec0c6121c970 2024-12-05T11:14:32,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/84aada0ea2d94d6c9df8ec0c6121c970, entries=150, sequenceid=531, filesize=12.0 K 2024-12-05T11:14:32,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=13.42 KB/13740 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1334ms, sequenceid=531, compaction requested=false 2024-12-05T11:14:32,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: 2024-12-05T11:14:33,182 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-05T11:14:33,182 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:33,182 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:33,182 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:33,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:33,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:14:33,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-05T11:14:33,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-05T11:14:33,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-05T11:14:33,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-12-05T11:14:33,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-12-05T11:14:33,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:14:33,184 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3772 2024-12-05T11:14:33,184 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3644 2024-12-05T11:14:33,184 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:14:33,184 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1675 2024-12-05T11:14:33,184 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5018 rows 2024-12-05T11:14:33,185 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1685 2024-12-05T11:14:33,185 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5049 rows 2024-12-05T11:14:33,185 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:14:33,185 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:14:33,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:33,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:33,186 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:33,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:14:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:14:33,201 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397273201"}]},"ts":"1733397273201"} 2024-12-05T11:14:33,204 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:14:33,204 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:14:33,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:14:33,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, UNASSIGN}] 2024-12-05T11:14:33,210 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, UNASSIGN 2024-12-05T11:14:33,211 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fb2f3b651a249ac1bc6c3ae631d8f7b9, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:33,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, UNASSIGN because future has completed 2024-12-05T11:14:33,214 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:14:33,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:14:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:14:33,371 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:33,371 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:14:33,372 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing fb2f3b651a249ac1bc6c3ae631d8f7b9, disabling compactions & flushes 2024-12-05T11:14:33,372 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:33,372 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:33,372 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. after waiting 0 ms 2024-12-05T11:14:33,372 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:33,373 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-05T11:14:33,373 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=A 2024-12-05T11:14:33,373 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:33,373 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=B 2024-12-05T11:14:33,373 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:33,373 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb2f3b651a249ac1bc6c3ae631d8f7b9, store=C 2024-12-05T11:14:33,373 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:33,379 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/302abc7eac764df59c94eafa30a270d6 is 50, key is test_row_2/A:col10/1733397273179/Put/seqid=0 2024-12-05T11:14:33,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741936_1112 (size=7415) 2024-12-05T11:14:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:14:33,786 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/302abc7eac764df59c94eafa30a270d6 2024-12-05T11:14:33,801 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/fa7ef19a008d40b68d672d21690fd460 is 50, key is test_row_2/B:col10/1733397273179/Put/seqid=0 2024-12-05T11:14:33,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741937_1113 (size=7415) 2024-12-05T11:14:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:14:34,207 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/fa7ef19a008d40b68d672d21690fd460 2024-12-05T11:14:34,219 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9ff02cc3febb422caf199014601a674f is 50, key is test_row_2/C:col10/1733397273179/Put/seqid=0 2024-12-05T11:14:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741938_1114 (size=7415) 2024-12-05T11:14:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:14:34,625 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9ff02cc3febb422caf199014601a674f 2024-12-05T11:14:34,637 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/A/302abc7eac764df59c94eafa30a270d6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/302abc7eac764df59c94eafa30a270d6 2024-12-05T11:14:34,643 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/302abc7eac764df59c94eafa30a270d6, entries=50, sequenceid=540, filesize=7.2 K 2024-12-05T11:14:34,644 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/B/fa7ef19a008d40b68d672d21690fd460 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/fa7ef19a008d40b68d672d21690fd460 2024-12-05T11:14:34,648 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/fa7ef19a008d40b68d672d21690fd460, entries=50, sequenceid=540, filesize=7.2 K 2024-12-05T11:14:34,649 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/.tmp/C/9ff02cc3febb422caf199014601a674f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9ff02cc3febb422caf199014601a674f 2024-12-05T11:14:34,654 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9ff02cc3febb422caf199014601a674f, entries=50, sequenceid=540, filesize=7.2 K 2024-12-05T11:14:34,655 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1283ms, sequenceid=540, compaction requested=true 2024-12-05T11:14:34,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/19886497ccf340db83c000c13519a694, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d6ba06bc5b854b1d96487719cc3c529c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/0ac0d59b0bf2468d8ba6e715fda2e845, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5bee48d0a1f140f9b0c5573a0786798a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/b2ea9e60c9fb4075946d256b851a94a2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7357e62b89ae409e9fee18a798cffd5b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/532cb9bae22d482aaacf2da7e69a3283, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/453b47ce97aa45619fefed595c0721f8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/2b012d4031f4441ea11f0a78dcd8be61, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/26491dd4567942eba728c91f5638c76d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/bd516ed0bc434089a6d5a435a29764e3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a5f3a1397bf4cb1aee7010eb44edf2a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/62940dad079944818cbf8a18cc05030d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c674526ae2b24a5c919da6f7533ceab0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/222c1d0b81a6404c9b5f14b9abc5f79b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/a1d5f562319448ba9b1582efc3a25706, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/38230db9681b40989cc3106299475eb6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/90a83d697ce8446ebcb787cf07089ced, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5fcacfbc5d964b9da4ba88f421d59de3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c9321af5d4cb40a6a21ad99fbf30f457, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/f65083079a1a466a9515a0a334bba59e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d196d90cc65d4c7092e4ac6acd1cb62e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/e052dcba41b64344900baff4e516b29c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a262d9d1da54317a8103ed3d6ba17d5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/9dac14d7680d45a7a2e9e38ef34abbc7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/23ee945d6e504e9e9e10e27ae0d5b1dd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/1d928abfc82a4aebba2045adc965d3cf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/550441ebce934c19bc05eaf74321dc9f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/be77d2b21459463d923633a966eb25a1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/8aa6469c40764aa69dd9312d8ba86330, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/fb71c328a3c448139bb70ba281162d52] to archive 2024-12-05T11:14:34,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:14:34,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/19886497ccf340db83c000c13519a694 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/19886497ccf340db83c000c13519a694 2024-12-05T11:14:34,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d6ba06bc5b854b1d96487719cc3c529c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d6ba06bc5b854b1d96487719cc3c529c 2024-12-05T11:14:34,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/0ac0d59b0bf2468d8ba6e715fda2e845 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/0ac0d59b0bf2468d8ba6e715fda2e845 2024-12-05T11:14:34,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5bee48d0a1f140f9b0c5573a0786798a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5bee48d0a1f140f9b0c5573a0786798a 2024-12-05T11:14:34,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/b2ea9e60c9fb4075946d256b851a94a2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/b2ea9e60c9fb4075946d256b851a94a2 2024-12-05T11:14:34,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7357e62b89ae409e9fee18a798cffd5b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7357e62b89ae409e9fee18a798cffd5b 2024-12-05T11:14:34,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/532cb9bae22d482aaacf2da7e69a3283 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/532cb9bae22d482aaacf2da7e69a3283 2024-12-05T11:14:34,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/453b47ce97aa45619fefed595c0721f8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/453b47ce97aa45619fefed595c0721f8 2024-12-05T11:14:34,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/2b012d4031f4441ea11f0a78dcd8be61 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/2b012d4031f4441ea11f0a78dcd8be61 2024-12-05T11:14:34,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/26491dd4567942eba728c91f5638c76d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/26491dd4567942eba728c91f5638c76d 2024-12-05T11:14:34,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/bd516ed0bc434089a6d5a435a29764e3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/bd516ed0bc434089a6d5a435a29764e3 2024-12-05T11:14:34,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a5f3a1397bf4cb1aee7010eb44edf2a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a5f3a1397bf4cb1aee7010eb44edf2a 2024-12-05T11:14:34,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/62940dad079944818cbf8a18cc05030d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/62940dad079944818cbf8a18cc05030d 2024-12-05T11:14:34,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c674526ae2b24a5c919da6f7533ceab0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c674526ae2b24a5c919da6f7533ceab0 2024-12-05T11:14:34,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/222c1d0b81a6404c9b5f14b9abc5f79b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/222c1d0b81a6404c9b5f14b9abc5f79b 2024-12-05T11:14:34,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/a1d5f562319448ba9b1582efc3a25706 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/a1d5f562319448ba9b1582efc3a25706 2024-12-05T11:14:34,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/38230db9681b40989cc3106299475eb6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/38230db9681b40989cc3106299475eb6 2024-12-05T11:14:34,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/90a83d697ce8446ebcb787cf07089ced to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/90a83d697ce8446ebcb787cf07089ced 2024-12-05T11:14:34,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5fcacfbc5d964b9da4ba88f421d59de3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/5fcacfbc5d964b9da4ba88f421d59de3 2024-12-05T11:14:34,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c9321af5d4cb40a6a21ad99fbf30f457 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/c9321af5d4cb40a6a21ad99fbf30f457 2024-12-05T11:14:34,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/f65083079a1a466a9515a0a334bba59e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/f65083079a1a466a9515a0a334bba59e 2024-12-05T11:14:34,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d196d90cc65d4c7092e4ac6acd1cb62e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/d196d90cc65d4c7092e4ac6acd1cb62e 2024-12-05T11:14:34,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/e052dcba41b64344900baff4e516b29c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/e052dcba41b64344900baff4e516b29c 2024-12-05T11:14:34,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a262d9d1da54317a8103ed3d6ba17d5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/7a262d9d1da54317a8103ed3d6ba17d5 2024-12-05T11:14:34,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/9dac14d7680d45a7a2e9e38ef34abbc7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/9dac14d7680d45a7a2e9e38ef34abbc7 2024-12-05T11:14:34,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/23ee945d6e504e9e9e10e27ae0d5b1dd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/23ee945d6e504e9e9e10e27ae0d5b1dd 2024-12-05T11:14:34,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/1d928abfc82a4aebba2045adc965d3cf to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/1d928abfc82a4aebba2045adc965d3cf 2024-12-05T11:14:34,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/550441ebce934c19bc05eaf74321dc9f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/550441ebce934c19bc05eaf74321dc9f 2024-12-05T11:14:34,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/be77d2b21459463d923633a966eb25a1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/be77d2b21459463d923633a966eb25a1 2024-12-05T11:14:34,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/8aa6469c40764aa69dd9312d8ba86330 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/8aa6469c40764aa69dd9312d8ba86330 2024-12-05T11:14:34,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/fb71c328a3c448139bb70ba281162d52 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/fb71c328a3c448139bb70ba281162d52 2024-12-05T11:14:34,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3267b8936dc64d15869def9f0689e12e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/94208a31c172407a93197715280c22c1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/e3c04b231d1448b38083ce14fd74573e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d1b24ed89e094643b010f0bb0d3d74d4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/c84567d97fa54030b07f6f93936f86bd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/2fdf6eec56b8403a86564a19a4d2ace0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/8c7c12e22fd742cab2716f677dab18bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7d997a2b75d2401e963942008d0aac9c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/6c87f4807c4d412baa89edc457993c40, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1d8ef90e065d42379df802f49561865f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b5012ec1881440f6b005d9d0dbf73cee, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3e31ec38330d4d758ca707f657725456, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/18023e32b3934313b0999eff0345d3f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3db4b9b6d24b457a850d4ccfd8316639, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1a871e3e8d67403a9f4dc6b7ac44ea18, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/0f0d68035f214ea787ee5d09579ed85c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d173674a91cb4bd0a35ed42f7d81c9f7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/bf3c4570ef684e06a4fb71d77eb2848d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7f1ea138bb2a4846a2de57537d47c071, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/956064382c204511868777e9aad93fe1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/5cbac49fdceb4a7dae1a41ba05466071, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/ae461f584fb54e46ac66f904fb48bfba, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3f2ff4356033470ab41b34e53db2cafc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3a908e757195423fbb3971ca9ac99b0d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/cd78f3e0dada45e4b2003795b0f6a560, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/aedaf657e10f4778a72556245050e769, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/a28a4369e7a54001b6696bdc7972c204, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/930a2160db7c413b9839734d7920b574, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b99ab81662c941e7b37bf5ace80963bd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/4af3d5276b3a45e4a66ff343edcb68a2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/77b2db0672054ff8a5e4e803d5652a14] to archive 2024-12-05T11:14:34,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:14:34,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3267b8936dc64d15869def9f0689e12e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3267b8936dc64d15869def9f0689e12e 2024-12-05T11:14:34,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/94208a31c172407a93197715280c22c1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/94208a31c172407a93197715280c22c1 2024-12-05T11:14:34,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/e3c04b231d1448b38083ce14fd74573e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/e3c04b231d1448b38083ce14fd74573e 2024-12-05T11:14:34,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d1b24ed89e094643b010f0bb0d3d74d4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d1b24ed89e094643b010f0bb0d3d74d4 2024-12-05T11:14:34,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/c84567d97fa54030b07f6f93936f86bd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/c84567d97fa54030b07f6f93936f86bd 2024-12-05T11:14:34,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/2fdf6eec56b8403a86564a19a4d2ace0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/2fdf6eec56b8403a86564a19a4d2ace0 2024-12-05T11:14:34,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/8c7c12e22fd742cab2716f677dab18bb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/8c7c12e22fd742cab2716f677dab18bb 2024-12-05T11:14:34,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7d997a2b75d2401e963942008d0aac9c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7d997a2b75d2401e963942008d0aac9c 2024-12-05T11:14:34,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/6c87f4807c4d412baa89edc457993c40 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/6c87f4807c4d412baa89edc457993c40 2024-12-05T11:14:34,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1d8ef90e065d42379df802f49561865f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1d8ef90e065d42379df802f49561865f 2024-12-05T11:14:34,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b5012ec1881440f6b005d9d0dbf73cee to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b5012ec1881440f6b005d9d0dbf73cee 2024-12-05T11:14:34,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3e31ec38330d4d758ca707f657725456 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3e31ec38330d4d758ca707f657725456 2024-12-05T11:14:34,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/18023e32b3934313b0999eff0345d3f9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/18023e32b3934313b0999eff0345d3f9 2024-12-05T11:14:34,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3db4b9b6d24b457a850d4ccfd8316639 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3db4b9b6d24b457a850d4ccfd8316639 2024-12-05T11:14:34,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1a871e3e8d67403a9f4dc6b7ac44ea18 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1a871e3e8d67403a9f4dc6b7ac44ea18 2024-12-05T11:14:34,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/0f0d68035f214ea787ee5d09579ed85c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/0f0d68035f214ea787ee5d09579ed85c 2024-12-05T11:14:34,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d173674a91cb4bd0a35ed42f7d81c9f7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/d173674a91cb4bd0a35ed42f7d81c9f7 2024-12-05T11:14:34,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/bf3c4570ef684e06a4fb71d77eb2848d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/bf3c4570ef684e06a4fb71d77eb2848d 2024-12-05T11:14:34,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7f1ea138bb2a4846a2de57537d47c071 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/7f1ea138bb2a4846a2de57537d47c071 2024-12-05T11:14:34,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/956064382c204511868777e9aad93fe1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/956064382c204511868777e9aad93fe1 2024-12-05T11:14:34,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/5cbac49fdceb4a7dae1a41ba05466071 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/5cbac49fdceb4a7dae1a41ba05466071 2024-12-05T11:14:34,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/ae461f584fb54e46ac66f904fb48bfba to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/ae461f584fb54e46ac66f904fb48bfba 2024-12-05T11:14:34,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3f2ff4356033470ab41b34e53db2cafc to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3f2ff4356033470ab41b34e53db2cafc 2024-12-05T11:14:34,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3a908e757195423fbb3971ca9ac99b0d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/3a908e757195423fbb3971ca9ac99b0d 2024-12-05T11:14:34,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/cd78f3e0dada45e4b2003795b0f6a560 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/cd78f3e0dada45e4b2003795b0f6a560 2024-12-05T11:14:34,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/aedaf657e10f4778a72556245050e769 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/aedaf657e10f4778a72556245050e769 2024-12-05T11:14:34,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/a28a4369e7a54001b6696bdc7972c204 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/a28a4369e7a54001b6696bdc7972c204 2024-12-05T11:14:34,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/930a2160db7c413b9839734d7920b574 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/930a2160db7c413b9839734d7920b574 2024-12-05T11:14:34,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b99ab81662c941e7b37bf5ace80963bd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/b99ab81662c941e7b37bf5ace80963bd 2024-12-05T11:14:34,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/4af3d5276b3a45e4a66ff343edcb68a2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/4af3d5276b3a45e4a66ff343edcb68a2 2024-12-05T11:14:34,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/77b2db0672054ff8a5e4e803d5652a14 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/77b2db0672054ff8a5e4e803d5652a14 2024-12-05T11:14:34,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/c35f54ddf3bf4be4a3d1b872f127b392, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6f90d99d983c4ba7bac46776bdf3fbf9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/bd1f091ffcdc4d35b7c5c53ce782a182, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/fe1d5b08d4584399b6598de799422b8f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f19b78acea92496888934ac52f91e369, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/65846ace328a4368adc8fef699d53719, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/efc579fc8fdc4c77bfc159d803a64845, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e280993fa7294102b559426f0a4b9c60, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/2e7572b991184531afb776f8a4d22051, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ddf951124c16440497265ac503673536, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/4613964f5e8c42129e5cf8d096496741, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e8340ea339e24dee9c65aa25a9dfbfe8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/23b72e11e17b4d5faa9d9ee766d1aa9c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/551ac8e8fb2642348f01a93da1787562, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/35e091a2a66d4af2b90169bd13d93e8d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/adf0b1d0e028444e8f5d430cee33cc60, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/126feeb9dce5481fa0b4f789cac77220, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac987818eca041e1a812855d34bfec35, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/b1a408392d80473a8ca199616798bf4e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6329d469e30b4d25a2e485756e80516e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ba57e0d2def14b338e8332465d01ed76, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/faaf72dbd7b14a27a857a72ca3fe0474, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9b91442ef9fe4fbbbada8c5d42d03196, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6b59113a08ce46d68d57ad5acf92b261, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/cee952928ac846528044a7d1db770ad1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f17176b7e6db419aadb10d4c0bfae24d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/d1e937dabc4f45f6af740f01e0a5d984, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/7b0785a0fffb43b18fe2c4bf90611927, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9710636d3b3e47f380468e11c82c8615, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/43eb9ae049024cd79ad99e1e07a7f5d0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac7ae8f637f74a1da37edcc44afea906] to archive 2024-12-05T11:14:34,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:14:34,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/c35f54ddf3bf4be4a3d1b872f127b392 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/c35f54ddf3bf4be4a3d1b872f127b392 2024-12-05T11:14:34,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6f90d99d983c4ba7bac46776bdf3fbf9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6f90d99d983c4ba7bac46776bdf3fbf9 2024-12-05T11:14:34,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/bd1f091ffcdc4d35b7c5c53ce782a182 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/bd1f091ffcdc4d35b7c5c53ce782a182 2024-12-05T11:14:34,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/fe1d5b08d4584399b6598de799422b8f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/fe1d5b08d4584399b6598de799422b8f 2024-12-05T11:14:34,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f19b78acea92496888934ac52f91e369 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f19b78acea92496888934ac52f91e369 2024-12-05T11:14:34,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/65846ace328a4368adc8fef699d53719 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/65846ace328a4368adc8fef699d53719 2024-12-05T11:14:34,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/efc579fc8fdc4c77bfc159d803a64845 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/efc579fc8fdc4c77bfc159d803a64845 2024-12-05T11:14:34,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e280993fa7294102b559426f0a4b9c60 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e280993fa7294102b559426f0a4b9c60 2024-12-05T11:14:34,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/2e7572b991184531afb776f8a4d22051 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/2e7572b991184531afb776f8a4d22051 2024-12-05T11:14:34,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ddf951124c16440497265ac503673536 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ddf951124c16440497265ac503673536 2024-12-05T11:14:34,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/4613964f5e8c42129e5cf8d096496741 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/4613964f5e8c42129e5cf8d096496741 2024-12-05T11:14:34,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e8340ea339e24dee9c65aa25a9dfbfe8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/e8340ea339e24dee9c65aa25a9dfbfe8 2024-12-05T11:14:34,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/23b72e11e17b4d5faa9d9ee766d1aa9c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/23b72e11e17b4d5faa9d9ee766d1aa9c 2024-12-05T11:14:34,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/551ac8e8fb2642348f01a93da1787562 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/551ac8e8fb2642348f01a93da1787562 2024-12-05T11:14:34,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/35e091a2a66d4af2b90169bd13d93e8d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/35e091a2a66d4af2b90169bd13d93e8d 2024-12-05T11:14:34,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/adf0b1d0e028444e8f5d430cee33cc60 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/adf0b1d0e028444e8f5d430cee33cc60 2024-12-05T11:14:34,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/126feeb9dce5481fa0b4f789cac77220 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/126feeb9dce5481fa0b4f789cac77220 2024-12-05T11:14:34,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac987818eca041e1a812855d34bfec35 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac987818eca041e1a812855d34bfec35 2024-12-05T11:14:34,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/b1a408392d80473a8ca199616798bf4e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/b1a408392d80473a8ca199616798bf4e 2024-12-05T11:14:34,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6329d469e30b4d25a2e485756e80516e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6329d469e30b4d25a2e485756e80516e 2024-12-05T11:14:34,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ba57e0d2def14b338e8332465d01ed76 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ba57e0d2def14b338e8332465d01ed76 2024-12-05T11:14:34,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/faaf72dbd7b14a27a857a72ca3fe0474 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/faaf72dbd7b14a27a857a72ca3fe0474 2024-12-05T11:14:34,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9b91442ef9fe4fbbbada8c5d42d03196 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9b91442ef9fe4fbbbada8c5d42d03196 2024-12-05T11:14:34,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6b59113a08ce46d68d57ad5acf92b261 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/6b59113a08ce46d68d57ad5acf92b261 2024-12-05T11:14:34,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/cee952928ac846528044a7d1db770ad1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/cee952928ac846528044a7d1db770ad1 2024-12-05T11:14:34,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f17176b7e6db419aadb10d4c0bfae24d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/f17176b7e6db419aadb10d4c0bfae24d 2024-12-05T11:14:34,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/d1e937dabc4f45f6af740f01e0a5d984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/d1e937dabc4f45f6af740f01e0a5d984 2024-12-05T11:14:34,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/7b0785a0fffb43b18fe2c4bf90611927 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/7b0785a0fffb43b18fe2c4bf90611927 2024-12-05T11:14:34,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9710636d3b3e47f380468e11c82c8615 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9710636d3b3e47f380468e11c82c8615 2024-12-05T11:14:34,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/43eb9ae049024cd79ad99e1e07a7f5d0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/43eb9ae049024cd79ad99e1e07a7f5d0 2024-12-05T11:14:34,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac7ae8f637f74a1da37edcc44afea906 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/ac7ae8f637f74a1da37edcc44afea906 2024-12-05T11:14:34,787 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/recovered.edits/543.seqid, newMaxSeqId=543, maxSeqId=1 2024-12-05T11:14:34,787 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. 2024-12-05T11:14:34,788 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for fb2f3b651a249ac1bc6c3ae631d8f7b9: Waiting for close lock at 1733397273372Running coprocessor pre-close hooks at 1733397273372Disabling compacts and flushes for region at 1733397273372Disabling writes for close at 1733397273372Obtaining lock to block concurrent updates at 1733397273373 (+1 ms)Preparing flush snapshotting stores in fb2f3b651a249ac1bc6c3ae631d8f7b9 at 1733397273373Finished memstore snapshotting TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733397273373Flushing stores of TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9. at 1733397273374 (+1 ms)Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/A: creating writer at 1733397273374Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/A: appending metadata at 1733397273378 (+4 ms)Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/A: closing flushed file at 1733397273378Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/B: creating writer at 1733397273797 (+419 ms)Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/B: appending metadata at 1733397273801 (+4 ms)Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/B: closing flushed file at 1733397273801Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/C: creating writer at 1733397274216 (+415 ms)Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/C: appending metadata at 1733397274218 (+2 ms)Flushing fb2f3b651a249ac1bc6c3ae631d8f7b9/C: closing flushed file at 1733397274218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@696419cb: reopening flushed file at 1733397274635 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48f8efd4: reopening flushed file at 1733397274643 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205664af: reopening flushed file at 1733397274649 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fb2f3b651a249ac1bc6c3ae631d8f7b9 in 1283ms, sequenceid=540, compaction requested=true at 1733397274655 (+6 ms)Writing region close event to WAL at 1733397274784 (+129 ms)Running coprocessor post-close hooks at 1733397274787 (+3 ms)Closed at 1733397274787 2024-12-05T11:14:34,790 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:34,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fb2f3b651a249ac1bc6c3ae631d8f7b9, regionState=CLOSED 2024-12-05T11:14:34,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:14:34,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T11:14:34,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure fb2f3b651a249ac1bc6c3ae631d8f7b9, server=6706a97ae65f,42931,1733397243639 in 1.5790 sec 2024-12-05T11:14:34,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-05T11:14:34,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb2f3b651a249ac1bc6c3ae631d8f7b9, UNASSIGN in 1.5860 sec 2024-12-05T11:14:34,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T11:14:34,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5930 sec 2024-12-05T11:14:34,801 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397274801"}]},"ts":"1733397274801"} 2024-12-05T11:14:34,802 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:14:34,802 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:14:34,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6090 sec 2024-12-05T11:14:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T11:14:35,350 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:35,355 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:14:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,363 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:14:35,364 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,371 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:35,377 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/recovered.edits] 2024-12-05T11:14:35,380 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/14bcf1517ec2468fb2f1cb01ea36baee to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/14bcf1517ec2468fb2f1cb01ea36baee 2024-12-05T11:14:35,381 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/302abc7eac764df59c94eafa30a270d6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/302abc7eac764df59c94eafa30a270d6 2024-12-05T11:14:35,382 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/eecdc05123464d99a786a64b5b3d2986 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/A/eecdc05123464d99a786a64b5b3d2986 2024-12-05T11:14:35,384 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1857af97449240b4b948050bcffb0b0d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1857af97449240b4b948050bcffb0b0d 2024-12-05T11:14:35,385 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1cb2276159ee4811a4a3b4cd0589e5c6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/1cb2276159ee4811a4a3b4cd0589e5c6 2024-12-05T11:14:35,387 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/fa7ef19a008d40b68d672d21690fd460 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/B/fa7ef19a008d40b68d672d21690fd460 2024-12-05T11:14:35,388 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/22bc4f73fd724919bd178bbb5d484d09 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/22bc4f73fd724919bd178bbb5d484d09 2024-12-05T11:14:35,389 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/84aada0ea2d94d6c9df8ec0c6121c970 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/84aada0ea2d94d6c9df8ec0c6121c970 2024-12-05T11:14:35,390 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9ff02cc3febb422caf199014601a674f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/C/9ff02cc3febb422caf199014601a674f 2024-12-05T11:14:35,394 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/recovered.edits/543.seqid to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9/recovered.edits/543.seqid 2024-12-05T11:14:35,394 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/fb2f3b651a249ac1bc6c3ae631d8f7b9 2024-12-05T11:14:35,395 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:14:35,400 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T11:14:35,410 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:14:35,438 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:14:35,440 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,440 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:14:35,440 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733397275440"}]},"ts":"9223372036854775807"} 2024-12-05T11:14:35,444 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:14:35,445 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fb2f3b651a249ac1bc6c3ae631d8f7b9, NAME => 'TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:14:35,445 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:14:35,445 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733397275445"}]},"ts":"9223372036854775807"} 2024-12-05T11:14:35,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:14:35,450 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-12-05T11:14:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T11:14:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:14:35,470 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:35,480 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;6706a97ae65f:42931-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2084105885_22 at /127.0.0.1:43210 [Waiting for operation #155] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_711469046_22 at /127.0.0.1:55046 [Waiting for operation #242] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_711469046_22 at /127.0.0.1:55206 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6184 (was 6737) 2024-12-05T11:14:35,488 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=6184 2024-12-05T11:14:35,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:14:35,490 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:14:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:35,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:14:35,492 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:35,492 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-05T11:14:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T11:14:35,492 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:14:35,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741939_1115 (size=1061) 2024-12-05T11:14:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T11:14:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T11:14:35,905 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:14:35,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741940_1116 (size=53) 2024-12-05T11:14:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T11:14:36,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:36,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b6afdaa222643745b7f25a1a31a4fff2, disabling compactions & flushes 2024-12-05T11:14:36,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. after waiting 0 ms 2024-12-05T11:14:36,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b6afdaa222643745b7f25a1a31a4fff2: Waiting for close lock at 1733397276313Disabling compacts and flushes for region at 1733397276313Disabling writes for close at 1733397276313Writing region close event to WAL at 1733397276313Closed at 1733397276313 2024-12-05T11:14:36,315 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:14:36,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733397276315"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733397276315"}]},"ts":"1733397276315"} 2024-12-05T11:14:36,318 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:14:36,319 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:14:36,319 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397276319"}]},"ts":"1733397276319"} 2024-12-05T11:14:36,322 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:14:36,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, ASSIGN}] 2024-12-05T11:14:36,324 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, ASSIGN 2024-12-05T11:14:36,325 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, ASSIGN; state=OFFLINE, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=false 2024-12-05T11:14:36,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:36,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, ASSIGN because future has completed 2024-12-05T11:14:36,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:14:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T11:14:36,640 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,641 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:14:36,641 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,641 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:36,641 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,641 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,642 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,644 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:36,644 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6afdaa222643745b7f25a1a31a4fff2 columnFamilyName A 2024-12-05T11:14:36,644 DEBUG [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:36,645 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(327): Store=b6afdaa222643745b7f25a1a31a4fff2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:36,645 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,646 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:36,646 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6afdaa222643745b7f25a1a31a4fff2 columnFamilyName B 2024-12-05T11:14:36,646 DEBUG [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:36,647 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(327): Store=b6afdaa222643745b7f25a1a31a4fff2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:36,647 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,648 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:36,648 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6afdaa222643745b7f25a1a31a4fff2 columnFamilyName C 2024-12-05T11:14:36,648 DEBUG [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:36,649 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(327): Store=b6afdaa222643745b7f25a1a31a4fff2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:36,649 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,650 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,650 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,651 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,651 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,652 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:14:36,653 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,655 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:14:36,655 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened b6afdaa222643745b7f25a1a31a4fff2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61850085, jitterRate=-0.07836191356182098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:14:36,655 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:36,656 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for b6afdaa222643745b7f25a1a31a4fff2: Running coprocessor pre-open hook at 1733397276641Writing region info on filesystem at 1733397276641Initializing all the Stores at 1733397276642 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397276642Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397276642Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397276642Cleaning up temporary data from old regions at 1733397276651 (+9 ms)Running coprocessor post-open hooks at 1733397276655 (+4 ms)Region opened successfully at 1733397276656 (+1 ms) 2024-12-05T11:14:36,657 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., pid=30, masterSystemTime=1733397276637 2024-12-05T11:14:36,658 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,659 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:36,659 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:36,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:14:36,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T11:14:36,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 in 178 msec 2024-12-05T11:14:36,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-05T11:14:36,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, ASSIGN in 342 msec 2024-12-05T11:14:36,666 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:14:36,666 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397276666"}]},"ts":"1733397276666"} 2024-12-05T11:14:36,668 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:14:36,669 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:14:36,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-05T11:14:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T11:14:37,639 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:37,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229ee4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:37,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:37,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:37,643 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa6213a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:37,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e15a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:37,648 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:37,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:37,654 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:14:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:14:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66520796 2024-12-05T11:14:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:14:37,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:14:37,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:14:37,661 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:14:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T11:14:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741941_1117 (size=1097) 2024-12-05T11:14:38,141 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T11:14:38,141 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T11:14:38,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:14:38,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, REOPEN/MOVE}] 2024-12-05T11:14:38,155 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, REOPEN/MOVE 2024-12-05T11:14:38,156 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, REOPEN/MOVE because future has completed 2024-12-05T11:14:38,158 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:14:38,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:14:38,311 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,312 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:14:38,312 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing b6afdaa222643745b7f25a1a31a4fff2, disabling compactions & flushes 2024-12-05T11:14:38,312 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,312 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,312 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. after waiting 0 ms 2024-12-05T11:14:38,312 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,321 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T11:14:38,322 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,322 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for b6afdaa222643745b7f25a1a31a4fff2: Waiting for close lock at 1733397278312Running coprocessor pre-close hooks at 1733397278312Disabling compacts and flushes for region at 1733397278312Disabling writes for close at 1733397278312Writing region close event to WAL at 1733397278313 (+1 ms)Running coprocessor post-close hooks at 1733397278322 (+9 ms)Closed at 1733397278322 2024-12-05T11:14:38,322 WARN [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: b6afdaa222643745b7f25a1a31a4fff2 to self. 2024-12-05T11:14:38,325 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,326 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=CLOSED 2024-12-05T11:14:38,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:14:38,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-05T11:14:38,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 in 171 msec 2024-12-05T11:14:38,333 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, REOPEN/MOVE; state=CLOSED, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=true 2024-12-05T11:14:38,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, REOPEN/MOVE because future has completed 2024-12-05T11:14:38,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:14:38,647 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,647 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:14:38,647 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,647 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:14:38,647 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,647 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,651 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,652 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:38,655 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6afdaa222643745b7f25a1a31a4fff2 columnFamilyName A 2024-12-05T11:14:38,657 DEBUG [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:38,658 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(327): Store=b6afdaa222643745b7f25a1a31a4fff2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:38,659 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,660 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:38,660 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6afdaa222643745b7f25a1a31a4fff2 columnFamilyName B 2024-12-05T11:14:38,660 DEBUG [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:38,660 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(327): Store=b6afdaa222643745b7f25a1a31a4fff2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:38,661 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,661 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:14:38,661 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6afdaa222643745b7f25a1a31a4fff2 columnFamilyName C 2024-12-05T11:14:38,661 DEBUG [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:38,662 INFO [StoreOpener-b6afdaa222643745b7f25a1a31a4fff2-1 {}] regionserver.HStore(327): Store=b6afdaa222643745b7f25a1a31a4fff2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:14:38,662 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,662 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,663 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,664 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,665 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,665 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,665 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:14:38,667 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,668 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened b6afdaa222643745b7f25a1a31a4fff2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74058906, jitterRate=0.10356369614601135}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:14:38,668 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,668 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for b6afdaa222643745b7f25a1a31a4fff2: Running coprocessor pre-open hook at 1733397278647Writing region info on filesystem at 1733397278647Initializing all the Stores at 1733397278648 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397278648Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397278648Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397278648Cleaning up temporary data from old regions at 1733397278665 (+17 ms)Running coprocessor post-open hooks at 1733397278668 (+3 ms)Region opened successfully at 1733397278668 2024-12-05T11:14:38,669 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., pid=35, masterSystemTime=1733397278644 2024-12-05T11:14:38,670 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,670 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,671 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=OPEN, openSeqNum=5, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:14:38,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-05T11:14:38,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 in 183 msec 2024-12-05T11:14:38,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-05T11:14:38,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, REOPEN/MOVE in 521 msec 2024-12-05T11:14:38,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T11:14:38,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-12-05T11:14:38,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0180 sec 2024-12-05T11:14:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T11:14:38,687 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:38,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280b3bfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,689 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61faae49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,691 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d9b2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70646c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,696 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3deebd7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580d088e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,698 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6414c893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,700 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b1d2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148de5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b75f253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9e8962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680235d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31423375, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,708 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4047d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721bdc4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,711 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bcb097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3627ac2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c4d062, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173af643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,716 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76fa478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aba8a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54894ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cd1319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362c2074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:14:38,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:14:38,722 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:14:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:14:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:14:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302c941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:14:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:14:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:38,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:14:38,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3649d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:14:38,725 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,725 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,725 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,725 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,725 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:38,726 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,727 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,727 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-05T11:14:38,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:38,728 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:38,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:38,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:38,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,731 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,738 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:14:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:14:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:14:38,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:14:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:14:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:38,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:14:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:38,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:38,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120516dfc22db1dd4fd9a2a543868c01e53e_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397278754/Put/seqid=0 2024-12-05T11:14:38,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741942_1118 (size=12154) 2024-12-05T11:14:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:38,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397338836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397338838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397338836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397338847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397338849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:38,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:38,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:38,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:38,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:38,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397338959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397338959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397338959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397338969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397338969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397339178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397339178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:39,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120516dfc22db1dd4fd9a2a543868c01e53e_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120516dfc22db1dd4fd9a2a543868c01e53e_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:39,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/512ae78cc32240ecb033954bba0daf0f, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:39,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/512ae78cc32240ecb033954bba0daf0f is 175, key is test_row_0/A:col10/1733397278754/Put/seqid=0 2024-12-05T11:14:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741943_1119 (size=30955) 2024-12-05T11:14:39,237 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/512ae78cc32240ecb033954bba0daf0f 2024-12-05T11:14:39,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c07b4e35e0a34922bf06b8ca9af882c2 is 50, key is test_row_0/B:col10/1733397278754/Put/seqid=0 2024-12-05T11:14:39,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741944_1120 (size=12001) 2024-12-05T11:14:39,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c07b4e35e0a34922bf06b8ca9af882c2 2024-12-05T11:14:39,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3922b2e3a73c49748e11adb1bcdc4fbb is 50, key is test_row_0/C:col10/1733397278754/Put/seqid=0 2024-12-05T11:14:39,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741945_1121 (size=12001) 2024-12-05T11:14:39,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3922b2e3a73c49748e11adb1bcdc4fbb 2024-12-05T11:14:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/512ae78cc32240ecb033954bba0daf0f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f 2024-12-05T11:14:39,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f, entries=150, sequenceid=16, filesize=30.2 K 2024-12-05T11:14:39,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c07b4e35e0a34922bf06b8ca9af882c2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c07b4e35e0a34922bf06b8ca9af882c2 2024-12-05T11:14:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:39,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c07b4e35e0a34922bf06b8ca9af882c2, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T11:14:39,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3922b2e3a73c49748e11adb1bcdc4fbb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3922b2e3a73c49748e11adb1bcdc4fbb 2024-12-05T11:14:39,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3922b2e3a73c49748e11adb1bcdc4fbb, entries=150, sequenceid=16, filesize=11.7 K 2024-12-05T11:14:39,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b6afdaa222643745b7f25a1a31a4fff2 in 620ms, sequenceid=16, compaction requested=false 2024-12-05T11:14:39,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:39,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T11:14:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:39,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397278835, max timestamp=1733397279479Num uniques 859; 2024-12-05T11:14:39,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397278835, max timestamp=1733397279479Num uniques 859; 2024-12-05T11:14:39,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:39,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:39,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397278835, max timestamp=1733397279479Num uniques 859; 2024-12-05T11:14:39,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T11:14:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:39,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:39,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:39,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:39,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:39,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053905430c2e204ef6824f54d3cc5d7d60_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397278843/Put/seqid=0 2024-12-05T11:14:39,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741946_1122 (size=17034) 2024-12-05T11:14:39,522 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397279512, max timestamp=1733397279524Num uniques 859; 2024-12-05T11:14:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397279512, max timestamp=1733397279524Num uniques 859; 2024-12-05T11:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397279512, max timestamp=1733397279524Num uniques 859; 2024-12-05T11:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:39,532 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053905430c2e204ef6824f54d3cc5d7d60_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053905430c2e204ef6824f54d3cc5d7d60_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:39,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/761e641c19ce4048ad95816a5279b73a, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:39,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/761e641c19ce4048ad95816a5279b73a is 175, key is test_row_0/A:col10/1733397278843/Put/seqid=0 2024-12-05T11:14:39,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741947_1123 (size=48139) 2024-12-05T11:14:39,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=39, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/761e641c19ce4048ad95816a5279b73a 2024-12-05T11:14:39,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/eb42ca6c15a94d55b70b6b5017742bf3 is 50, key is test_row_0/B:col10/1733397278843/Put/seqid=0 2024-12-05T11:14:39,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741948_1124 (size=12001) 2024-12-05T11:14:39,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397279583, max timestamp=1733397279589Num uniques 1018; 2024-12-05T11:14:39,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:14:39,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397279583, max timestamp=1733397279589Num uniques 1018; 2024-12-05T11:14:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-05T11:14:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397279583, max timestamp=1733397279589Num uniques 1018; 2024-12-05T11:14:39,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:14:39,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733397279622, max timestamp=1733397279634Num uniques 1027; 2024-12-05T11:14:39,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733397279622, max timestamp=1733397279634Num uniques 1009; 2024-12-05T11:14:39,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-05T11:14:39,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-12-05T11:14:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733397279622, max timestamp=1733397279634Num uniques 1027; 2024-12-05T11:14:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-05T11:14:39,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733397279646, max timestamp=1733397279663Num uniques 1086; 2024-12-05T11:14:39,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733397279646, max timestamp=1733397279663Num uniques 1086; 2024-12-05T11:14:39,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=17676, suffix heap size=140912,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=386 2024-12-05T11:14:39,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=17676, suffix heap size=140912,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=386 2024-12-05T11:14:39,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1733397279646, max timestamp=1733397279663Num uniques 1086; 2024-12-05T11:14:39,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=17676, suffix heap size=140912,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=386 2024-12-05T11:14:39,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397339690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397339690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397339691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397339692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397339692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397339808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397339808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397339808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397339809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397339810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:39,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:39,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:39,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:39,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:39,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/eb42ca6c15a94d55b70b6b5017742bf3 2024-12-05T11:14:39,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3bd48046e733446a8a921214b53db9bc is 50, key is test_row_0/C:col10/1733397278843/Put/seqid=0 2024-12-05T11:14:40,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741949_1125 (size=12001) 2024-12-05T11:14:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397340018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397340019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397340025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397340025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397340025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:40,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:40,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,315 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:14:40,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397340339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397340339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397340340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397340340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397340340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3bd48046e733446a8a921214b53db9bc 2024-12-05T11:14:40,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/761e641c19ce4048ad95816a5279b73a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a 2024-12-05T11:14:40,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a, entries=250, sequenceid=39, filesize=47.0 K 2024-12-05T11:14:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/eb42ca6c15a94d55b70b6b5017742bf3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/eb42ca6c15a94d55b70b6b5017742bf3 2024-12-05T11:14:40,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:40,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/eb42ca6c15a94d55b70b6b5017742bf3, entries=150, sequenceid=39, filesize=11.7 K 2024-12-05T11:14:40,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3bd48046e733446a8a921214b53db9bc as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3bd48046e733446a8a921214b53db9bc 2024-12-05T11:14:40,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3bd48046e733446a8a921214b53db9bc, entries=150, sequenceid=39, filesize=11.7 K 2024-12-05T11:14:40,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=154.31 KB/158010 for b6afdaa222643745b7f25a1a31a4fff2 in 970ms, sequenceid=39, compaction requested=false 2024-12-05T11:14:40,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:40,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T11:14:40,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:40,580 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=154.31 KB heapSize=423.47 KB 2024-12-05T11:14:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:40,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:40,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205abad732ea0aa419ead3da07cf4223c9b_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397279688/Put/seqid=0 2024-12-05T11:14:40,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741950_1126 (size=12154) 2024-12-05T11:14:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:40,632 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205abad732ea0aa419ead3da07cf4223c9b_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205abad732ea0aa419ead3da07cf4223c9b_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:40,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7b0229c076e84afe87712771703e0376, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7b0229c076e84afe87712771703e0376 is 175, key is test_row_0/A:col10/1733397279688/Put/seqid=0 2024-12-05T11:14:40,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741951_1127 (size=30955) 2024-12-05T11:14:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397340857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397340859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397340859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397340859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397340859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397340968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397340968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,043 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7b0229c076e84afe87712771703e0376 2024-12-05T11:14:41,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/734206cc2fd64da891acb6eab2868fa0 is 50, key is test_row_0/B:col10/1733397279688/Put/seqid=0 2024-12-05T11:14:41,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741952_1128 (size=12001) 2024-12-05T11:14:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397341178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397341178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,461 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/734206cc2fd64da891acb6eab2868fa0 2024-12-05T11:14:41,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/83ee3f9e2aba4f87a7ec7adfced5efb6 is 50, key is test_row_0/C:col10/1733397279688/Put/seqid=0 2024-12-05T11:14:41,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741953_1129 (size=12001) 2024-12-05T11:14:41,480 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/83ee3f9e2aba4f87a7ec7adfced5efb6 2024-12-05T11:14:41,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7b0229c076e84afe87712771703e0376 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376 2024-12-05T11:14:41,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397341489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397341489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,497 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376, entries=150, sequenceid=126, filesize=30.2 K 2024-12-05T11:14:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/734206cc2fd64da891acb6eab2868fa0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/734206cc2fd64da891acb6eab2868fa0 2024-12-05T11:14:41,505 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/734206cc2fd64da891acb6eab2868fa0, entries=150, sequenceid=126, filesize=11.7 K 2024-12-05T11:14:41,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/83ee3f9e2aba4f87a7ec7adfced5efb6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/83ee3f9e2aba4f87a7ec7adfced5efb6 2024-12-05T11:14:41,511 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/83ee3f9e2aba4f87a7ec7adfced5efb6, entries=150, sequenceid=126, filesize=11.7 K 2024-12-05T11:14:41,512 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~423.42 KB/433584, currentSize=33.54 KB/34350 for b6afdaa222643745b7f25a1a31a4fff2 in 931ms, sequenceid=126, compaction requested=true 2024-12-05T11:14:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-05T11:14:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-05T11:14:41,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-05T11:14:41,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7840 sec 2024-12-05T11:14:41,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.7920 sec 2024-12-05T11:14:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:41,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:41,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fcc1bc63b07f453dad7d06f39e5f3f7a_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:41,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741954_1130 (size=14794) 2024-12-05T11:14:41,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:41,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fcc1bc63b07f453dad7d06f39e5f3f7a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fcc1bc63b07f453dad7d06f39e5f3f7a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:41,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e84d411a0a1a46d999fe69616e26e2f2, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:41,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e84d411a0a1a46d999fe69616e26e2f2 is 175, key is test_row_0/A:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:41,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733397281897, max timestamp=1733397281916Num uniques 859; 2024-12-05T11:14:41,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733397281897, max timestamp=1733397281916Num uniques 859; 2024-12-05T11:14:41,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733397281897, max timestamp=1733397281916Num uniques 859; 2024-12-05T11:14:41,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T11:14:41,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T11:14:41,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T11:14:41,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741955_1131 (size=39749) 2024-12-05T11:14:41,930 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e84d411a0a1a46d999fe69616e26e2f2 2024-12-05T11:14:41,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/3d651d5fc81f46498633762782c66bfc is 50, key is test_row_0/B:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397341949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741956_1132 (size=12151) 2024-12-05T11:14:41,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/3d651d5fc81f46498633762782c66bfc 2024-12-05T11:14:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397341958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397341958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:41,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c20be753a1cb48a0b7835f591a0d0216 is 50, key is test_row_0/C:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:41,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741957_1133 (size=12151) 2024-12-05T11:14:41,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c20be753a1cb48a0b7835f591a0d0216 2024-12-05T11:14:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e84d411a0a1a46d999fe69616e26e2f2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2 2024-12-05T11:14:42,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2, entries=200, sequenceid=136, filesize=38.8 K 2024-12-05T11:14:42,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/3d651d5fc81f46498633762782c66bfc as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/3d651d5fc81f46498633762782c66bfc 2024-12-05T11:14:42,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397341998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397341998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/3d651d5fc81f46498633762782c66bfc, entries=150, sequenceid=136, filesize=11.9 K 2024-12-05T11:14:42,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c20be753a1cb48a0b7835f591a0d0216 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c20be753a1cb48a0b7835f591a0d0216 2024-12-05T11:14:42,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c20be753a1cb48a0b7835f591a0d0216, entries=150, sequenceid=136, filesize=11.9 K 2024-12-05T11:14:42,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for b6afdaa222643745b7f25a1a31a4fff2 in 140ms, sequenceid=136, compaction requested=true 2024-12-05T11:14:42,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:42,017 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:42,017 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:42,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:42,018 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:42,018 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:42,018 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:14:42,018 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:14:42,018 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:42,018 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:42,019 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=146.3 K 2024-12-05T11:14:42,019 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c07b4e35e0a34922bf06b8ca9af882c2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/eb42ca6c15a94d55b70b6b5017742bf3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/734206cc2fd64da891acb6eab2868fa0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/3d651d5fc81f46498633762782c66bfc] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=47.0 K 2024-12-05T11:14:42,019 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:42,019 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2] 2024-12-05T11:14:42,019 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c07b4e35e0a34922bf06b8ca9af882c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733397278753 2024-12-05T11:14:42,020 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512ae78cc32240ecb033954bba0daf0f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733397278753 2024-12-05T11:14:42,020 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting eb42ca6c15a94d55b70b6b5017742bf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1733397278843 2024-12-05T11:14:42,020 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761e641c19ce4048ad95816a5279b73a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1733397278835 2024-12-05T11:14:42,020 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 734206cc2fd64da891acb6eab2868fa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733397279688 2024-12-05T11:14:42,021 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0229c076e84afe87712771703e0376, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733397279688 2024-12-05T11:14:42,021 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d651d5fc81f46498633762782c66bfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733397280857 2024-12-05T11:14:42,021 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e84d411a0a1a46d999fe69616e26e2f2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733397280855 2024-12-05T11:14:42,033 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:42,034 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/4f48db40350d4883bccf24466f3a2fc7 is 50, key is test_row_0/B:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:42,036 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:42,045 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205dc2bb60bdf684aa8a54cad44ec2b5065_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:42,053 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205dc2bb60bdf684aa8a54cad44ec2b5065_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:42,053 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dc2bb60bdf684aa8a54cad44ec2b5065_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:42,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741958_1134 (size=12289) 2024-12-05T11:14:42,069 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/4f48db40350d4883bccf24466f3a2fc7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/4f48db40350d4883bccf24466f3a2fc7 2024-12-05T11:14:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:42,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-05T11:14:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:42,082 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into 4f48db40350d4883bccf24466f3a2fc7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:42,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:42,082 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=12, startTime=1733397282017; duration=0sec 2024-12-05T11:14:42,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:42,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:14:42,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:42,084 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:42,084 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:14:42,085 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:42,085 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3922b2e3a73c49748e11adb1bcdc4fbb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3bd48046e733446a8a921214b53db9bc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/83ee3f9e2aba4f87a7ec7adfced5efb6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c20be753a1cb48a0b7835f591a0d0216] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=47.0 K 2024-12-05T11:14:42,086 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3922b2e3a73c49748e11adb1bcdc4fbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733397278753 2024-12-05T11:14:42,086 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd48046e733446a8a921214b53db9bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1733397278843 2024-12-05T11:14:42,086 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ee3f9e2aba4f87a7ec7adfced5efb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733397279688 2024-12-05T11:14:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397342080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397342081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,088 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c20be753a1cb48a0b7835f591a0d0216, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733397280857 2024-12-05T11:14:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397342082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205279443abe10449b680674d98e2997b5d_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397281954/Put/seqid=0 2024-12-05T11:14:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741959_1135 (size=4469) 2024-12-05T11:14:42,108 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#115 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:42,109 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7cdcae28cd6849f1b2257c0584b4617a is 175, key is test_row_0/A:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:42,119 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:42,120 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c3f511bbcdeb4ec29ae0ad8446467bbd is 50, key is test_row_0/C:col10/1733397281875/Put/seqid=0 2024-12-05T11:14:42,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741961_1137 (size=31243) 2024-12-05T11:14:42,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741960_1136 (size=14794) 2024-12-05T11:14:42,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741962_1138 (size=12289) 2024-12-05T11:14:42,140 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7cdcae28cd6849f1b2257c0584b4617a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cdcae28cd6849f1b2257c0584b4617a 2024-12-05T11:14:42,153 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c3f511bbcdeb4ec29ae0ad8446467bbd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c3f511bbcdeb4ec29ae0ad8446467bbd 2024-12-05T11:14:42,157 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into 7cdcae28cd6849f1b2257c0584b4617a(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:42,157 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:42,157 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=12, startTime=1733397282017; duration=0sec 2024-12-05T11:14:42,157 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:42,157 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:14:42,161 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into c3f511bbcdeb4ec29ae0ad8446467bbd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:42,161 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:42,161 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=12, startTime=1733397282017; duration=0sec 2024-12-05T11:14:42,162 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:42,162 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:14:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397342198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397342199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397342199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397342409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397342409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397342409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:42,538 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205279443abe10449b680674d98e2997b5d_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205279443abe10449b680674d98e2997b5d_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:42,539 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7cc93344d5b3495ca675cd29963b2bc8, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:42,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7cc93344d5b3495ca675cd29963b2bc8 is 175, key is test_row_0/A:col10/1733397281954/Put/seqid=0 2024-12-05T11:14:42,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741963_1139 (size=39749) 2024-12-05T11:14:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397342718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397342729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397342729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T11:14:42,889 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:42,889 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-05T11:14:42,891 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T11:14:42,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:42,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:42,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7cc93344d5b3495ca675cd29963b2bc8 2024-12-05T11:14:42,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c385b3a75d86435bb7c0668e9e912c41 is 50, key is test_row_0/B:col10/1733397281954/Put/seqid=0 2024-12-05T11:14:42,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741964_1140 (size=12151) 2024-12-05T11:14:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T11:14:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397343018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397343018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:43,020 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:43,020 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:43,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T11:14:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T11:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T11:14:43,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397343229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:43,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397343238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:43,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:43,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397343248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:43,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T11:14:43,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:43,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:43,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c385b3a75d86435bb7c0668e9e912c41 2024-12-05T11:14:43,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/52e5f5a01503460583f68d20b33ab0a9 is 50, key is test_row_0/C:col10/1733397281954/Put/seqid=0 2024-12-05T11:14:43,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741965_1141 (size=12151) 2024-12-05T11:14:43,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/52e5f5a01503460583f68d20b33ab0a9 2024-12-05T11:14:43,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/7cc93344d5b3495ca675cd29963b2bc8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8 2024-12-05T11:14:43,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8, entries=200, sequenceid=173, filesize=38.8 K 2024-12-05T11:14:43,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c385b3a75d86435bb7c0668e9e912c41 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c385b3a75d86435bb7c0668e9e912c41 2024-12-05T11:14:43,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c385b3a75d86435bb7c0668e9e912c41, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:14:43,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/52e5f5a01503460583f68d20b33ab0a9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/52e5f5a01503460583f68d20b33ab0a9 2024-12-05T11:14:43,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/52e5f5a01503460583f68d20b33ab0a9, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:14:43,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for b6afdaa222643745b7f25a1a31a4fff2 in 1342ms, sequenceid=173, compaction requested=false 2024-12-05T11:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:43,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T11:14:43,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:43,504 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T11:14:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:43,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120558b938fd5b1a49ccab340c6f2b49806a_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397282081/Put/seqid=0 2024-12-05T11:14:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T11:14:43,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741966_1142 (size=12304) 2024-12-05T11:14:43,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:43,930 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120558b938fd5b1a49ccab340c6f2b49806a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120558b938fd5b1a49ccab340c6f2b49806a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f55d5add319449ce850f20e12ef41282, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:43,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f55d5add319449ce850f20e12ef41282 is 175, key is test_row_0/A:col10/1733397282081/Put/seqid=0 2024-12-05T11:14:43,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741967_1143 (size=31105) 2024-12-05T11:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T11:14:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:44,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:44,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733397284264, max timestamp=1733397284274Num uniques 859; 2024-12-05T11:14:44,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733397284264, max timestamp=1733397284274Num uniques 859; 2024-12-05T11:14:44,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733397284264, max timestamp=1733397284274Num uniques 859; 2024-12-05T11:14:44,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-05T11:14:44,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-05T11:14:44,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-05T11:14:44,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397344298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397344300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397344303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,338 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f55d5add319449ce850f20e12ef41282 2024-12-05T11:14:44,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/9026b4874602490abe065185b37a4e95 is 50, key is test_row_0/B:col10/1733397282081/Put/seqid=0 2024-12-05T11:14:44,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741968_1144 (size=12151) 2024-12-05T11:14:44,350 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/9026b4874602490abe065185b37a4e95 2024-12-05T11:14:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/a209a6c73de341e29d1e7f3d3509e43a is 50, key is test_row_0/C:col10/1733397282081/Put/seqid=0 2024-12-05T11:14:44,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741969_1145 (size=12151) 2024-12-05T11:14:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397344408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397344408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397344418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397344619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397344619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397344628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,763 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/a209a6c73de341e29d1e7f3d3509e43a 2024-12-05T11:14:44,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f55d5add319449ce850f20e12ef41282 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282 2024-12-05T11:14:44,773 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282, entries=150, sequenceid=184, filesize=30.4 K 2024-12-05T11:14:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/9026b4874602490abe065185b37a4e95 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/9026b4874602490abe065185b37a4e95 2024-12-05T11:14:44,779 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/9026b4874602490abe065185b37a4e95, entries=150, sequenceid=184, filesize=11.9 K 2024-12-05T11:14:44,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/a209a6c73de341e29d1e7f3d3509e43a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a209a6c73de341e29d1e7f3d3509e43a 2024-12-05T11:14:44,785 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a209a6c73de341e29d1e7f3d3509e43a, entries=150, sequenceid=184, filesize=11.9 K 2024-12-05T11:14:44,786 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for b6afdaa222643745b7f25a1a31a4fff2 in 1282ms, sequenceid=184, compaction requested=true 2024-12-05T11:14:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-05T11:14:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-05T11:14:44,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-05T11:14:44,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8960 sec 2024-12-05T11:14:44,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.9000 sec 2024-12-05T11:14:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:44,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=161.02 KB heapSize=446.39 KB 2024-12-05T11:14:44,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:44,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:14:44,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:44,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:14:44,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:44,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:14:44,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053d196c7d3ec649d682660617658df76b_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397344938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397344938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397344938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:44,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741970_1146 (size=17284) 2024-12-05T11:14:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T11:14:45,039 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:45,039 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-05T11:14:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:45,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397345038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:45,042 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:45,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397345049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397345049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397345049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,052 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:45,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:45,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:45,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397345259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397345259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:45,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,352 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053d196c7d3ec649d682660617658df76b_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053d196c7d3ec649d682660617658df76b_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:45,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2a9c236ed2834ded95948c5aa50274db, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:45,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2a9c236ed2834ded95948c5aa50274db is 175, key is test_row_0/A:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:45,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741971_1147 (size=48389) 2024-12-05T11:14:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:45,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397345448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397345568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397345568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:45,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:45,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:45,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:45,758 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2a9c236ed2834ded95948c5aa50274db 2024-12-05T11:14:45,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/cb3df922288243fdbad871435b66d05b is 50, key is test_row_0/B:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:45,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741972_1148 (size=12151) 2024-12-05T11:14:45,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:45,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:45,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:45,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:45,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:45,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397346079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397346079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:46,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:46,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:46,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:46,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/cb3df922288243fdbad871435b66d05b 2024-12-05T11:14:46,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/d264740d91ad42bdbc76cb3c43a9f7b2 is 50, key is test_row_0/C:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741973_1149 (size=12151) 2024-12-05T11:14:46,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:46,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:46,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397346458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:46,459 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:46,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:46,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:46,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:46,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/d264740d91ad42bdbc76cb3c43a9f7b2 2024-12-05T11:14:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2a9c236ed2834ded95948c5aa50274db as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db 2024-12-05T11:14:46,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db, entries=250, sequenceid=222, filesize=47.3 K 2024-12-05T11:14:46,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/cb3df922288243fdbad871435b66d05b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/cb3df922288243fdbad871435b66d05b 2024-12-05T11:14:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/cb3df922288243fdbad871435b66d05b, entries=150, sequenceid=222, filesize=11.9 K 2024-12-05T11:14:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/d264740d91ad42bdbc76cb3c43a9f7b2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d264740d91ad42bdbc76cb3c43a9f7b2 2024-12-05T11:14:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d264740d91ad42bdbc76cb3c43a9f7b2, entries=150, sequenceid=222, filesize=11.9 K 2024-12-05T11:14:46,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~466.50 KB/477696, currentSize=20.13 KB/20610 for b6afdaa222643745b7f25a1a31a4fff2 in 1690ms, sequenceid=222, compaction requested=true 2024-12-05T11:14:46,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:46,623 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:46,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:46,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:46,623 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:46,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:46,625 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:46,625 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:14:46,625 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,625 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/4f48db40350d4883bccf24466f3a2fc7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c385b3a75d86435bb7c0668e9e912c41, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/9026b4874602490abe065185b37a4e95, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/cb3df922288243fdbad871435b66d05b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=47.6 K 2024-12-05T11:14:46,625 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150486 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:46,626 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:14:46,626 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,626 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cdcae28cd6849f1b2257c0584b4617a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=147.0 K 2024-12-05T11:14:46,626 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,626 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cdcae28cd6849f1b2257c0584b4617a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db] 2024-12-05T11:14:46,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:46,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:46,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:46,627 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f48db40350d4883bccf24466f3a2fc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733397280857 2024-12-05T11:14:46,627 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cdcae28cd6849f1b2257c0584b4617a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733397280857 2024-12-05T11:14:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,627 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc93344d5b3495ca675cd29963b2bc8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397281945 2024-12-05T11:14:46,627 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c385b3a75d86435bb7c0668e9e912c41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397281945 2024-12-05T11:14:46,627 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55d5add319449ce850f20e12ef41282, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733397282072 2024-12-05T11:14:46,627 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 9026b4874602490abe065185b37a4e95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733397282072 2024-12-05T11:14:46,628 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a9c236ed2834ded95948c5aa50274db, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397284295 2024-12-05T11:14:46,628 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3df922288243fdbad871435b66d05b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397284302 2024-12-05T11:14:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,638 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,643 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#127 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,644 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c532b6f463dd44dfa4fee8fad3cb9f23 is 50, key is test_row_0/B:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,645 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205a178b656c7a047edb950f6f0e760d036_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,651 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205a178b656c7a047edb950f6f0e760d036_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:46,652 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a178b656c7a047edb950f6f0e760d036_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741974_1150 (size=12425) 2024-12-05T11:14:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741975_1151 (size=4469) 2024-12-05T11:14:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,679 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#126 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,680 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/0c3d438e9c08402093591ef56f34e892 is 175, key is test_row_0/A:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741976_1152 (size=31379) 2024-12-05T11:14:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T11:14:46,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:46,733 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-05T11:14:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ac22cc10352c450ca2a4b8a4c730f6bd_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_1/A:col10/1733397284937/Put/seqid=0 2024-12-05T11:14:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741977_1153 (size=9814) 2024-12-05T11:14:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,075 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/c532b6f463dd44dfa4fee8fad3cb9f23 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c532b6f463dd44dfa4fee8fad3cb9f23 2024-12-05T11:14:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,080 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into c532b6f463dd44dfa4fee8fad3cb9f23(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:47,080 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,081 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=12, startTime=1733397286623; duration=0sec 2024-12-05T11:14:47,081 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:47,081 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:14:47,081 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:14:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:14:47,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:14:47,083 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:47,083 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c3f511bbcdeb4ec29ae0ad8446467bbd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/52e5f5a01503460583f68d20b33ab0a9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a209a6c73de341e29d1e7f3d3509e43a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d264740d91ad42bdbc76cb3c43a9f7b2] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=47.6 K 2024-12-05T11:14:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,083 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c3f511bbcdeb4ec29ae0ad8446467bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733397280857 2024-12-05T11:14:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,084 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 52e5f5a01503460583f68d20b33ab0a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397281945 2024-12-05T11:14:47,084 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting a209a6c73de341e29d1e7f3d3509e43a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733397282072 2024-12-05T11:14:47,084 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d264740d91ad42bdbc76cb3c43a9f7b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397284302 2024-12-05T11:14:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,093 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/0c3d438e9c08402093591ef56f34e892 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/0c3d438e9c08402093591ef56f34e892 2024-12-05T11:14:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,095 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,097 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/aa5ac87933dc4212aebd6b13665a6aa0 is 50, key is test_row_0/C:col10/1733397284302/Put/seqid=0 2024-12-05T11:14:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,103 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into 0c3d438e9c08402093591ef56f34e892(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:47,103 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,103 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=12, startTime=1733397286623; duration=0sec 2024-12-05T11:14:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,103 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:47,103 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:14:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741978_1154 (size=12425) 2024-12-05T11:14:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,119 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/aa5ac87933dc4212aebd6b13665a6aa0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/aa5ac87933dc4212aebd6b13665a6aa0 2024-12-05T11:14:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,125 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into aa5ac87933dc4212aebd6b13665a6aa0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:47,126 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,126 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=12, startTime=1733397286626; duration=0sec 2024-12-05T11:14:47,126 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,126 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:14:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:47,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,177 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ac22cc10352c450ca2a4b8a4c730f6bd_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ac22cc10352c450ca2a4b8a4c730f6bd_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397287163, max timestamp=1733397287170Num uniques 859; 2024-12-05T11:14:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397287163, max timestamp=1733397287170Num uniques 859; 2024-12-05T11:14:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397287163, max timestamp=1733397287170Num uniques 859; 2024-12-05T11:14:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:14:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ec7efef26a647a2929b06d673a2af9e, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ec7efef26a647a2929b06d673a2af9e is 175, key is test_row_1/A:col10/1733397284937/Put/seqid=0 2024-12-05T11:14:47,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741979_1155 (size=22461) 2024-12-05T11:14:47,188 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ec7efef26a647a2929b06d673a2af9e 2024-12-05T11:14:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:47,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/b52136e7b51346c5985ad4cffe23f1b2 is 50, key is test_row_1/B:col10/1733397284937/Put/seqid=0 2024-12-05T11:14:47,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741980_1156 (size=9757) 2024-12-05T11:14:47,203 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/b52136e7b51346c5985ad4cffe23f1b2 2024-12-05T11:14:47,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/2627e6a3247144a68b5781ad80875bad is 50, key is test_row_1/C:col10/1733397284937/Put/seqid=0 2024-12-05T11:14:47,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741981_1157 (size=9757) 2024-12-05T11:14:47,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397287214, max timestamp=1733397287220Num uniques 1018; 2024-12-05T11:14:47,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:14:47,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397287214, max timestamp=1733397287220Num uniques 1018; 2024-12-05T11:14:47,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:14:47,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397287214, max timestamp=1733397287220Num uniques 1018; 2024-12-05T11:14:47,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:14:47,227 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/2627e6a3247144a68b5781ad80875bad 2024-12-05T11:14:47,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ec7efef26a647a2929b06d673a2af9e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e 2024-12-05T11:14:47,239 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e, entries=100, sequenceid=228, filesize=21.9 K 2024-12-05T11:14:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/b52136e7b51346c5985ad4cffe23f1b2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b52136e7b51346c5985ad4cffe23f1b2 2024-12-05T11:14:47,250 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b52136e7b51346c5985ad4cffe23f1b2, entries=100, sequenceid=228, filesize=9.5 K 2024-12-05T11:14:47,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/2627e6a3247144a68b5781ad80875bad as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/2627e6a3247144a68b5781ad80875bad 2024-12-05T11:14:47,259 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/2627e6a3247144a68b5781ad80875bad, entries=100, sequenceid=228, filesize=9.5 K 2024-12-05T11:14:47,261 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=114.05 KB/116790 for b6afdaa222643745b7f25a1a31a4fff2 in 528ms, sequenceid=228, compaction requested=false 2024-12-05T11:14:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-05T11:14:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-05T11:14:47,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=127.47 KB heapSize=364.22 KB 2024-12-05T11:14:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:47,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:47,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-05T11:14:47,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2210 sec 2024-12-05T11:14:47,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2270 sec 2024-12-05T11:14:47,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205741925c6f02a44318fdc790d55229833_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741982_1158 (size=14994) 2024-12-05T11:14:47,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205741925c6f02a44318fdc790d55229833_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205741925c6f02a44318fdc790d55229833_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:47,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/1451fac580f2473891119bf138c537f3, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/1451fac580f2473891119bf138c537f3 is 175, key is test_row_0/A:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397347287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397347287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741983_1159 (size=39949) 2024-12-05T11:14:47,308 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/1451fac580f2473891119bf138c537f3 2024-12-05T11:14:47,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/e4fbe82db38942f4be939dafd10f90e7 is 50, key is test_row_0/B:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741984_1160 (size=12301) 2024-12-05T11:14:47,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/e4fbe82db38942f4be939dafd10f90e7 2024-12-05T11:14:47,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/75fa4f7b2f46425586aa4d0106684fbf is 50, key is test_row_0/C:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741985_1161 (size=12301) 2024-12-05T11:14:47,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/75fa4f7b2f46425586aa4d0106684fbf 2024-12-05T11:14:47,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/1451fac580f2473891119bf138c537f3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3 2024-12-05T11:14:47,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3, entries=200, sequenceid=284, filesize=39.0 K 2024-12-05T11:14:47,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/e4fbe82db38942f4be939dafd10f90e7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/e4fbe82db38942f4be939dafd10f90e7 2024-12-05T11:14:47,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/e4fbe82db38942f4be939dafd10f90e7, entries=150, sequenceid=284, filesize=12.0 K 2024-12-05T11:14:47,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/75fa4f7b2f46425586aa4d0106684fbf as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/75fa4f7b2f46425586aa4d0106684fbf 2024-12-05T11:14:47,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/75fa4f7b2f46425586aa4d0106684fbf, entries=150, sequenceid=284, filesize=12.0 K 2024-12-05T11:14:47,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~364.17 KB/372912, currentSize=60.38 KB/61830 for b6afdaa222643745b7f25a1a31a4fff2 in 125ms, sequenceid=284, compaction requested=true 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:47,388 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:47,388 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:47,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:47,389 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:47,389 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:14:47,389 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:47,389 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:47,389 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:14:47,389 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:47,389 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/0c3d438e9c08402093591ef56f34e892, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=91.6 K 2024-12-05T11:14:47,389 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c532b6f463dd44dfa4fee8fad3cb9f23, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b52136e7b51346c5985ad4cffe23f1b2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/e4fbe82db38942f4be939dafd10f90e7] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=33.7 K 2024-12-05T11:14:47,389 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:47,390 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/0c3d438e9c08402093591ef56f34e892, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3] 2024-12-05T11:14:47,390 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c532b6f463dd44dfa4fee8fad3cb9f23, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397284302 2024-12-05T11:14:47,392 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b52136e7b51346c5985ad4cffe23f1b2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733397284936 2024-12-05T11:14:47,392 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3d438e9c08402093591ef56f34e892, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397284302 2024-12-05T11:14:47,392 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e4fbe82db38942f4be939dafd10f90e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733397287240 2024-12-05T11:14:47,392 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ec7efef26a647a2929b06d673a2af9e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733397284936 2024-12-05T11:14:47,393 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1451fac580f2473891119bf138c537f3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733397287240 2024-12-05T11:14:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:47,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:14:47,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:47,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:47,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:47,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:47,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:47,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:47,406 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:47,406 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb is 50, key is test_row_0/B:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,408 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,417 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412051547689b6bc34d45b6334fb3c8759f44_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,419 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412051547689b6bc34d45b6334fb3c8759f44_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,419 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051547689b6bc34d45b6334fb3c8759f44_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056c283e51af914fe7a2b3c84b97996597_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397287400/Put/seqid=0 2024-12-05T11:14:47,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397347436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397347440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741986_1162 (size=12677) 2024-12-05T11:14:47,470 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb 2024-12-05T11:14:47,478 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into 5c2c9bf6b36c4a6b8b2c60d4d4a4edbb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:47,478 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,478 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=13, startTime=1733397287388; duration=0sec 2024-12-05T11:14:47,478 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:47,478 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:14:47,478 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741987_1163 (size=4469) 2024-12-05T11:14:47,482 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:47,482 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:14:47,482 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:47,483 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/aa5ac87933dc4212aebd6b13665a6aa0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/2627e6a3247144a68b5781ad80875bad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/75fa4f7b2f46425586aa4d0106684fbf] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=33.7 K 2024-12-05T11:14:47,483 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#136 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:47,484 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/1c0023e36f9f4d4fa0f776143bddf53f is 175, key is test_row_0/A:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,484 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5ac87933dc4212aebd6b13665a6aa0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397284302 2024-12-05T11:14:47,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 2627e6a3247144a68b5781ad80875bad, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733397284936 2024-12-05T11:14:47,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 75fa4f7b2f46425586aa4d0106684fbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733397287240 2024-12-05T11:14:47,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741988_1164 (size=14994) 2024-12-05T11:14:47,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741989_1165 (size=31631) 2024-12-05T11:14:47,548 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#138 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:47,549 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/1c0023e36f9f4d4fa0f776143bddf53f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1c0023e36f9f4d4fa0f776143bddf53f 2024-12-05T11:14:47,549 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3b56fb3499d1470b8a4d91034aaac7bf is 50, key is test_row_0/C:col10/1733397287256/Put/seqid=0 2024-12-05T11:14:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397347549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397347549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,559 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into 1c0023e36f9f4d4fa0f776143bddf53f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:47,559 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,559 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=13, startTime=1733397287388; duration=0sec 2024-12-05T11:14:47,559 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:47,559 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:14:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741990_1166 (size=12677) 2024-12-05T11:14:47,592 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/3b56fb3499d1470b8a4d91034aaac7bf as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3b56fb3499d1470b8a4d91034aaac7bf 2024-12-05T11:14:47,601 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into 3b56fb3499d1470b8a4d91034aaac7bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:47,601 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:47,601 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=13, startTime=1733397287388; duration=0sec 2024-12-05T11:14:47,601 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:47,601 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:14:47,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397347758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397347758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:47,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:47,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056c283e51af914fe7a2b3c84b97996597_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056c283e51af914fe7a2b3c84b97996597_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:47,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/41521f8a211d48f28bfbd6fab61a4ab6, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:47,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/41521f8a211d48f28bfbd6fab61a4ab6 is 175, key is test_row_0/A:col10/1733397287400/Put/seqid=0 2024-12-05T11:14:47,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741991_1167 (size=39949) 2024-12-05T11:14:48,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397348069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:48,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397348069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:48,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/41521f8a211d48f28bfbd6fab61a4ab6 2024-12-05T11:14:48,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/45e5099dab954e42a2a36683efd63701 is 50, key is test_row_0/B:col10/1733397287400/Put/seqid=0 2024-12-05T11:14:48,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741992_1168 (size=12301) 2024-12-05T11:14:48,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/45e5099dab954e42a2a36683efd63701 2024-12-05T11:14:48,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/80a5e1535a49413bb43e13a9956af0c6 is 50, key is test_row_0/C:col10/1733397287400/Put/seqid=0 2024-12-05T11:14:48,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741993_1169 (size=12301) 2024-12-05T11:14:48,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397348468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:48,472 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397348578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397348578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:48,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/80a5e1535a49413bb43e13a9956af0c6 2024-12-05T11:14:48,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/41521f8a211d48f28bfbd6fab61a4ab6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6 2024-12-05T11:14:48,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6, entries=200, sequenceid=298, filesize=39.0 K 2024-12-05T11:14:48,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/45e5099dab954e42a2a36683efd63701 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/45e5099dab954e42a2a36683efd63701 2024-12-05T11:14:48,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/45e5099dab954e42a2a36683efd63701, entries=150, sequenceid=298, filesize=12.0 K 2024-12-05T11:14:48,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/80a5e1535a49413bb43e13a9956af0c6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/80a5e1535a49413bb43e13a9956af0c6 2024-12-05T11:14:48,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/80a5e1535a49413bb43e13a9956af0c6, entries=150, sequenceid=298, filesize=12.0 K 2024-12-05T11:14:48,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b6afdaa222643745b7f25a1a31a4fff2 in 1456ms, sequenceid=298, compaction requested=false 2024-12-05T11:14:48,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:48,937 INFO [master/6706a97ae65f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T11:14:48,937 INFO [master/6706a97ae65f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T11:14:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:49,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:14:49,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:49,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:49,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:49,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:49,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:49,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:49,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205af2e8c43c6e5424b9b90aca1d1a8ff12_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:49,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741994_1170 (size=14994) 2024-12-05T11:14:49,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:49,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205af2e8c43c6e5424b9b90aca1d1a8ff12_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205af2e8c43c6e5424b9b90aca1d1a8ff12_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:49,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2b684782e94949a7b3226d02c67a0589, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:49,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2b684782e94949a7b3226d02c67a0589 is 175, key is test_row_0/A:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397349101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397349104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741995_1171 (size=39949) 2024-12-05T11:14:49,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2b684782e94949a7b3226d02c67a0589 2024-12-05T11:14:49,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/265a9598a4cf4aa78b403cc53a01473b is 50, key is test_row_0/B:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741996_1172 (size=12301) 2024-12-05T11:14:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T11:14:49,209 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:49,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397349208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-05T11:14:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:49,212 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:49,213 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:49,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397349219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:49,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:14:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397349418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397349428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:14:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:49,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/265a9598a4cf4aa78b403cc53a01473b 2024-12-05T11:14:49,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/6f21271c4b234b98b582c47d72917e8c is 50, key is test_row_0/C:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:49,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741997_1173 (size=12301) 2024-12-05T11:14:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397349588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,592 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397349598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,602 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:14:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397349728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397349738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:49,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:14:49,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:49,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:49,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:14:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:49,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/6f21271c4b234b98b582c47d72917e8c 2024-12-05T11:14:49,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:49,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2b684782e94949a7b3226d02c67a0589 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589 2024-12-05T11:14:49,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589, entries=200, sequenceid=320, filesize=39.0 K 2024-12-05T11:14:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/265a9598a4cf4aa78b403cc53a01473b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/265a9598a4cf4aa78b403cc53a01473b 2024-12-05T11:14:49,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/265a9598a4cf4aa78b403cc53a01473b, entries=150, sequenceid=320, filesize=12.0 K 2024-12-05T11:14:49,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/6f21271c4b234b98b582c47d72917e8c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/6f21271c4b234b98b582c47d72917e8c 2024-12-05T11:14:50,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/6f21271c4b234b98b582c47d72917e8c, entries=150, sequenceid=320, filesize=12.0 K 2024-12-05T11:14:50,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b6afdaa222643745b7f25a1a31a4fff2 in 933ms, sequenceid=320, compaction requested=true 2024-12-05T11:14:50,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:50,005 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:50,006 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:50,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:50,007 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111529 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:50,007 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:14:50,007 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:50,007 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:50,007 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:14:50,007 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1c0023e36f9f4d4fa0f776143bddf53f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=108.9 K 2024-12-05T11:14:50,007 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:50,007 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:50,007 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/45e5099dab954e42a2a36683efd63701, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/265a9598a4cf4aa78b403cc53a01473b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.4 K 2024-12-05T11:14:50,007 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1c0023e36f9f4d4fa0f776143bddf53f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589] 2024-12-05T11:14:50,008 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c0023e36f9f4d4fa0f776143bddf53f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733397287240 2024-12-05T11:14:50,008 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2c9bf6b36c4a6b8b2c60d4d4a4edbb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733397287240 2024-12-05T11:14:50,008 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41521f8a211d48f28bfbd6fab61a4ab6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733397287275 2024-12-05T11:14:50,008 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e5099dab954e42a2a36683efd63701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733397287275 2024-12-05T11:14:50,008 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 265a9598a4cf4aa78b403cc53a01473b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733397287430 2024-12-05T11:14:50,008 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b684782e94949a7b3226d02c67a0589, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733397287430 2024-12-05T11:14:50,025 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:50,028 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:50,028 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120541ea1bdc7f724a45aa2b52dc306007ee_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:50,028 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/1658a6072aa047659b623ad6e77a4c66 is 50, key is test_row_0/B:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:50,030 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120541ea1bdc7f724a45aa2b52dc306007ee_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:50,030 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120541ea1bdc7f724a45aa2b52dc306007ee_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:50,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741998_1174 (size=12779) 2024-12-05T11:14:50,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741999_1175 (size=4469) 2024-12-05T11:14:50,041 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#144 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:50,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e012ced140564f78ba655205f91e6c76 is 175, key is test_row_0/A:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:50,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742000_1176 (size=31733) 2024-12-05T11:14:50,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T11:14:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:50,131 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:14:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:50,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c21987b1c82f44079f7de7805bce7c6f_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397289095/Put/seqid=0 2024-12-05T11:14:50,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742001_1177 (size=12454) 2024-12-05T11:14:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:50,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397350270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397350270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:50,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397350379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397350388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,443 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/1658a6072aa047659b623ad6e77a4c66 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/1658a6072aa047659b623ad6e77a4c66 2024-12-05T11:14:50,449 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into 1658a6072aa047659b623ad6e77a4c66(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:50,449 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:50,449 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=13, startTime=1733397290006; duration=0sec 2024-12-05T11:14:50,449 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:50,449 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:14:50,449 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:50,451 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:50,451 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:14:50,451 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:50,451 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3b56fb3499d1470b8a4d91034aaac7bf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/80a5e1535a49413bb43e13a9956af0c6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/6f21271c4b234b98b582c47d72917e8c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.4 K 2024-12-05T11:14:50,452 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b56fb3499d1470b8a4d91034aaac7bf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733397287240 2024-12-05T11:14:50,453 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a5e1535a49413bb43e13a9956af0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733397287275 2024-12-05T11:14:50,454 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f21271c4b234b98b582c47d72917e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733397287430 2024-12-05T11:14:50,456 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e012ced140564f78ba655205f91e6c76 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e012ced140564f78ba655205f91e6c76 2024-12-05T11:14:50,462 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into e012ced140564f78ba655205f91e6c76(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:50,462 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:50,462 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=13, startTime=1733397290005; duration=0sec 2024-12-05T11:14:50,462 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:50,462 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:14:50,463 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:50,463 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/12d19d33b9bc4955beddc19251acd447 is 50, key is test_row_0/C:col10/1733397287433/Put/seqid=0 2024-12-05T11:14:50,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742002_1178 (size=12779) 2024-12-05T11:14:50,489 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/12d19d33b9bc4955beddc19251acd447 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/12d19d33b9bc4955beddc19251acd447 2024-12-05T11:14:50,499 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into 12d19d33b9bc4955beddc19251acd447(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:50,499 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:50,499 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=13, startTime=1733397290006; duration=0sec 2024-12-05T11:14:50,499 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:50,499 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:14:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:50,552 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c21987b1c82f44079f7de7805bce7c6f_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c21987b1c82f44079f7de7805bce7c6f_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:50,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/12b87c3b920d47348e0e71a76fcbd387, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:50,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/12b87c3b920d47348e0e71a76fcbd387 is 175, key is test_row_0/A:col10/1733397289095/Put/seqid=0 2024-12-05T11:14:50,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742003_1179 (size=31255) 2024-12-05T11:14:50,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397350588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397350599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397350898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397350908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:50,973 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/12b87c3b920d47348e0e71a76fcbd387 2024-12-05T11:14:50,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5e25854f53334007b155b2f4c6bd88b8 is 50, key is test_row_0/B:col10/1733397289095/Put/seqid=0 2024-12-05T11:14:50,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742004_1180 (size=12301) 2024-12-05T11:14:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:51,388 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5e25854f53334007b155b2f4c6bd88b8 2024-12-05T11:14:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c624c8e80285441eb730f82eea8f81aa is 50, key is test_row_0/C:col10/1733397289095/Put/seqid=0 2024-12-05T11:14:51,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742005_1181 (size=12301) 2024-12-05T11:14:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397351418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397351418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:51,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397351609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:51,611 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397351619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:51,620 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:51,800 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c624c8e80285441eb730f82eea8f81aa 2024-12-05T11:14:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/12b87c3b920d47348e0e71a76fcbd387 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387 2024-12-05T11:14:51,814 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387, entries=150, sequenceid=333, filesize=30.5 K 2024-12-05T11:14:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5e25854f53334007b155b2f4c6bd88b8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5e25854f53334007b155b2f4c6bd88b8 2024-12-05T11:14:51,824 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5e25854f53334007b155b2f4c6bd88b8, entries=150, sequenceid=333, filesize=12.0 K 2024-12-05T11:14:51,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/c624c8e80285441eb730f82eea8f81aa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c624c8e80285441eb730f82eea8f81aa 2024-12-05T11:14:51,841 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c624c8e80285441eb730f82eea8f81aa, entries=150, sequenceid=333, filesize=12.0 K 2024-12-05T11:14:51,849 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b6afdaa222643745b7f25a1a31a4fff2 in 1717ms, sequenceid=333, compaction requested=false 2024-12-05T11:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-05T11:14:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-05T11:14:51,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T11:14:51,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-12-05T11:14:51,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6480 sec 2024-12-05T11:14:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:52,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:14:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:52,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e38e701f1884430db99a82546d75c11a_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:52,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742006_1182 (size=17534) 2024-12-05T11:14:52,446 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397352447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397352448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e38e701f1884430db99a82546d75c11a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e38e701f1884430db99a82546d75c11a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:52,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ae5a5da7f104da2b719550f1285b251, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:52,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ae5a5da7f104da2b719550f1285b251 is 175, key is test_row_0/A:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:52,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742007_1183 (size=48639) 2024-12-05T11:14:52,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36034 deadline: 1733397352498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,499 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397352558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397352558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397352768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397352768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:52,871 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ae5a5da7f104da2b719550f1285b251 2024-12-05T11:14:52,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/7b03e49d3b2f4623ab32203ee4c2dc0e is 50, key is test_row_0/B:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:52,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742008_1184 (size=12301) 2024-12-05T11:14:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397353078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:53,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397353078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:53,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/7b03e49d3b2f4623ab32203ee4c2dc0e 2024-12-05T11:14:53,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/a8a0bf8ef5224c75be19e878f8cbb676 is 50, key is test_row_0/C:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742009_1185 (size=12301) 2024-12-05T11:14:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T11:14:53,379 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:53,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-05T11:14:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:14:53,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:53,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:53,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:14:53,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T11:14:53,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:53,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:53,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397353588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397353588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:53,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T11:14:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:14:53,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/a8a0bf8ef5224c75be19e878f8cbb676 2024-12-05T11:14:53,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ae5a5da7f104da2b719550f1285b251 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251 2024-12-05T11:14:53,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251, entries=250, sequenceid=357, filesize=47.5 K 2024-12-05T11:14:53,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/7b03e49d3b2f4623ab32203ee4c2dc0e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/7b03e49d3b2f4623ab32203ee4c2dc0e 2024-12-05T11:14:53,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/7b03e49d3b2f4623ab32203ee4c2dc0e, entries=150, sequenceid=357, filesize=12.0 K 2024-12-05T11:14:53,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/a8a0bf8ef5224c75be19e878f8cbb676 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a8a0bf8ef5224c75be19e878f8cbb676 2024-12-05T11:14:53,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a8a0bf8ef5224c75be19e878f8cbb676, entries=150, sequenceid=357, filesize=12.0 K 2024-12-05T11:14:53,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for b6afdaa222643745b7f25a1a31a4fff2 in 1294ms, sequenceid=357, compaction requested=true 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:53,724 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:53,724 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:53,725 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:53,725 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:53,725 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:14:53,725 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:14:53,725 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,725 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,725 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e012ced140564f78ba655205f91e6c76, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=109.0 K 2024-12-05T11:14:53,725 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/1658a6072aa047659b623ad6e77a4c66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5e25854f53334007b155b2f4c6bd88b8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/7b03e49d3b2f4623ab32203ee4c2dc0e] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.5 K 2024-12-05T11:14:53,726 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,726 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e012ced140564f78ba655205f91e6c76, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251] 2024-12-05T11:14:53,726 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1658a6072aa047659b623ad6e77a4c66, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733397287430 2024-12-05T11:14:53,726 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e012ced140564f78ba655205f91e6c76, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733397287430 2024-12-05T11:14:53,726 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b87c3b920d47348e0e71a76fcbd387, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733397289095 2024-12-05T11:14:53,726 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e25854f53334007b155b2f4c6bd88b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733397289095 2024-12-05T11:14:53,726 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae5a5da7f104da2b719550f1285b251, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733397290257 2024-12-05T11:14:53,727 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b03e49d3b2f4623ab32203ee4c2dc0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733397290257 2024-12-05T11:14:53,734 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:53,736 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:53,737 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/46e7d93cc08647e8a275eb92adf597f7 is 50, key is test_row_0/B:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:53,737 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120575b1a77b4bcd4bdbb930b5e464ffbdf3_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:53,739 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120575b1a77b4bcd4bdbb930b5e464ffbdf3_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:53,739 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120575b1a77b4bcd4bdbb930b5e464ffbdf3_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:53,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742010_1186 (size=12881) 2024-12-05T11:14:53,747 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/46e7d93cc08647e8a275eb92adf597f7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/46e7d93cc08647e8a275eb92adf597f7 2024-12-05T11:14:53,754 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into 46e7d93cc08647e8a275eb92adf597f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:53,754 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:53,754 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=13, startTime=1733397293724; duration=0sec 2024-12-05T11:14:53,754 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:53,754 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:14:53,754 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:53,756 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:53,757 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:14:53,757 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,757 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/12d19d33b9bc4955beddc19251acd447, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c624c8e80285441eb730f82eea8f81aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a8a0bf8ef5224c75be19e878f8cbb676] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.5 K 2024-12-05T11:14:53,758 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d19d33b9bc4955beddc19251acd447, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733397287430 2024-12-05T11:14:53,758 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c624c8e80285441eb730f82eea8f81aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733397289095 2024-12-05T11:14:53,759 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a0bf8ef5224c75be19e878f8cbb676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733397290257 2024-12-05T11:14:53,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742011_1187 (size=4469) 2024-12-05T11:14:53,767 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:53,767 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/fa4e19fdf24f40ac8b4fe4985a3de43d is 50, key is test_row_0/C:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:53,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742012_1188 (size=12881) 2024-12-05T11:14:53,779 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/fa4e19fdf24f40ac8b4fe4985a3de43d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/fa4e19fdf24f40ac8b4fe4985a3de43d 2024-12-05T11:14:53,785 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into fa4e19fdf24f40ac8b4fe4985a3de43d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:53,785 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:53,785 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=13, startTime=1733397293724; duration=0sec 2024-12-05T11:14:53,785 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:53,786 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:14:53,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T11:14:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:53,841 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:14:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:53,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058b685be3f35a4f7ba2463b7907d7ce72_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397292446/Put/seqid=0 2024-12-05T11:14:53,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742013_1189 (size=12454) 2024-12-05T11:14:53,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:53,859 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058b685be3f35a4f7ba2463b7907d7ce72_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058b685be3f35a4f7ba2463b7907d7ce72_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f5ad65955eb2417b909e0b6c56593807, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f5ad65955eb2417b909e0b6c56593807 is 175, key is test_row_0/A:col10/1733397292446/Put/seqid=0 2024-12-05T11:14:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742014_1190 (size=31255) 2024-12-05T11:14:53,869 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f5ad65955eb2417b909e0b6c56593807 2024-12-05T11:14:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/2a8a60a422e8459b8685b9c34f55733b is 50, key is test_row_0/B:col10/1733397292446/Put/seqid=0 2024-12-05T11:14:53,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742015_1191 (size=12301) 2024-12-05T11:14:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:14:54,162 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:54,163 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/050936b167d64be9b1dc66bf02657651 is 175, key is test_row_0/A:col10/1733397292430/Put/seqid=0 2024-12-05T11:14:54,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742016_1192 (size=31835) 2024-12-05T11:14:54,283 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/2a8a60a422e8459b8685b9c34f55733b 2024-12-05T11:14:54,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/04af4ea17706485db24f1e8b5d7e0460 is 50, key is test_row_0/C:col10/1733397292446/Put/seqid=0 2024-12-05T11:14:54,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742017_1193 (size=12301) 2024-12-05T11:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:14:54,572 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/050936b167d64be9b1dc66bf02657651 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/050936b167d64be9b1dc66bf02657651 2024-12-05T11:14:54,577 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into 050936b167d64be9b1dc66bf02657651(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:54,577 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:54,577 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=13, startTime=1733397293724; duration=0sec 2024-12-05T11:14:54,577 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:54,577 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:14:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:54,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:54,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397354625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:54,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397294612, max timestamp=1733397294627Num uniques 859; 2024-12-05T11:14:54,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397294612, max timestamp=1733397294627Num uniques 859; 2024-12-05T11:14:54,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397294612, max timestamp=1733397294627Num uniques 859; 2024-12-05T11:14:54,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:14:54,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:14:54,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:14:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397354668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:54,694 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/04af4ea17706485db24f1e8b5d7e0460 2024-12-05T11:14:54,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/f5ad65955eb2417b909e0b6c56593807 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807 2024-12-05T11:14:54,703 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807, entries=150, sequenceid=370, filesize=30.5 K 2024-12-05T11:14:54,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/2a8a60a422e8459b8685b9c34f55733b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/2a8a60a422e8459b8685b9c34f55733b 2024-12-05T11:14:54,709 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/2a8a60a422e8459b8685b9c34f55733b, entries=150, sequenceid=370, filesize=12.0 K 2024-12-05T11:14:54,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/04af4ea17706485db24f1e8b5d7e0460 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/04af4ea17706485db24f1e8b5d7e0460 2024-12-05T11:14:54,715 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/04af4ea17706485db24f1e8b5d7e0460, entries=150, sequenceid=370, filesize=12.0 K 2024-12-05T11:14:54,715 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b6afdaa222643745b7f25a1a31a4fff2 in 875ms, sequenceid=370, compaction requested=false 2024-12-05T11:14:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-05T11:14:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-05T11:14:54,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T11:14:54,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3360 sec 2024-12-05T11:14:54,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3400 sec 2024-12-05T11:14:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:54,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T11:14:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:54,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:54,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205122dcd22de2a4d4e892fdbe5822bce0a_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:54,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742018_1194 (size=14994) 2024-12-05T11:14:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397354766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397354778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:54,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397354878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:54,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397354988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397355088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:55,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205122dcd22de2a4d4e892fdbe5822bce0a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205122dcd22de2a4d4e892fdbe5822bce0a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:55,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e5bb4e4129674616b140845bb08f3fbc, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:55,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e5bb4e4129674616b140845bb08f3fbc is 175, key is test_row_0/A:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742019_1195 (size=39949) 2024-12-05T11:14:55,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397355298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397355398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T11:14:55,528 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:55,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:14:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-05T11:14:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:55,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:14:55,531 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:14:55,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:14:55,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e5bb4e4129674616b140845bb08f3fbc 2024-12-05T11:14:55,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/06023d28cad743dfa873c038ff323206 is 50, key is test_row_0/B:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:55,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742020_1196 (size=12301) 2024-12-05T11:14:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36016 deadline: 1733397355638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,639 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35998 deadline: 1733397355648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,651 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:14:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:55,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397355808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:14:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:55,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397355908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:55,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/06023d28cad743dfa873c038ff323206 2024-12-05T11:14:55,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/d19f736682444ff9a07a1537122f93ec is 50, key is test_row_0/C:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:55,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:14:55,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:55,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:55,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:55,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:55,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742021_1197 (size=12301) 2024-12-05T11:14:56,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:14:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:56,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:14:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:14:56,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/d19f736682444ff9a07a1537122f93ec 2024-12-05T11:14:56,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/e5bb4e4129674616b140845bb08f3fbc as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc 2024-12-05T11:14:56,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc, entries=200, sequenceid=407, filesize=39.0 K 2024-12-05T11:14:56,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/06023d28cad743dfa873c038ff323206 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/06023d28cad743dfa873c038ff323206 2024-12-05T11:14:56,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/06023d28cad743dfa873c038ff323206, entries=150, sequenceid=407, filesize=12.0 K 2024-12-05T11:14:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/d19f736682444ff9a07a1537122f93ec as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d19f736682444ff9a07a1537122f93ec 2024-12-05T11:14:56,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d19f736682444ff9a07a1537122f93ec, entries=150, sequenceid=407, filesize=12.0 K 2024-12-05T11:14:56,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for b6afdaa222643745b7f25a1a31a4fff2 in 1677ms, sequenceid=407, compaction requested=true 2024-12-05T11:14:56,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:56,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:14:56,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:56,418 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:56,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:14:56,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:56,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:14:56,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:56,419 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:56,420 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:56,420 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:56,420 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:14:56,420 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:14:56,420 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,420 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,420 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/46e7d93cc08647e8a275eb92adf597f7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/2a8a60a422e8459b8685b9c34f55733b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/06023d28cad743dfa873c038ff323206] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.6 K 2024-12-05T11:14:56,420 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/050936b167d64be9b1dc66bf02657651, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=100.6 K 2024-12-05T11:14:56,420 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,420 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/050936b167d64be9b1dc66bf02657651, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc] 2024-12-05T11:14:56,421 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050936b167d64be9b1dc66bf02657651, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733397290257 2024-12-05T11:14:56,421 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e7d93cc08647e8a275eb92adf597f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733397290257 2024-12-05T11:14:56,422 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8a60a422e8459b8685b9c34f55733b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733397292438 2024-12-05T11:14:56,422 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ad65955eb2417b909e0b6c56593807, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733397292438 2024-12-05T11:14:56,422 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 06023d28cad743dfa873c038ff323206, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733397294665 2024-12-05T11:14:56,422 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5bb4e4129674616b140845bb08f3fbc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733397294658 2024-12-05T11:14:56,446 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:56,447 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:56,448 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/35d48fec19ca47b79e043d39c25a28ed is 50, key is test_row_0/B:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:56,451 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053cdabe6d54e3441ea87f45c2040b3ad5_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:56,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T11:14:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,451 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:14:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:14:56,453 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053cdabe6d54e3441ea87f45c2040b3ad5_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:56,453 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053cdabe6d54e3441ea87f45c2040b3ad5_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:56,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742022_1198 (size=12983) 2024-12-05T11:14:56,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b8901cdd0cf0474cb569f694b80d5491_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397294744/Put/seqid=0 2024-12-05T11:14:56,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742023_1199 (size=4469) 2024-12-05T11:14:56,478 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#163 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:56,478 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a is 175, key is test_row_0/A:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:56,480 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/35d48fec19ca47b79e043d39c25a28ed as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/35d48fec19ca47b79e043d39c25a28ed 2024-12-05T11:14:56,486 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into 35d48fec19ca47b79e043d39c25a28ed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:56,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:56,486 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=13, startTime=1733397296418; duration=0sec 2024-12-05T11:14:56,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:14:56,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:14:56,487 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:14:56,488 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:14:56,488 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:14:56,488 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:56,488 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/fa4e19fdf24f40ac8b4fe4985a3de43d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/04af4ea17706485db24f1e8b5d7e0460, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d19f736682444ff9a07a1537122f93ec] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.6 K 2024-12-05T11:14:56,489 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4e19fdf24f40ac8b4fe4985a3de43d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733397290257 2024-12-05T11:14:56,490 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 04af4ea17706485db24f1e8b5d7e0460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733397292438 2024-12-05T11:14:56,490 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d19f736682444ff9a07a1537122f93ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733397294665 2024-12-05T11:14:56,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742024_1200 (size=12454) 2024-12-05T11:14:56,505 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:14:56,505 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/25caac8a2fc84e84aee5013c585d2efc is 50, key is test_row_0/C:col10/1733397294667/Put/seqid=0 2024-12-05T11:14:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742025_1201 (size=31937) 2024-12-05T11:14:56,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742026_1202 (size=12983) 2024-12-05T11:14:56,518 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/25caac8a2fc84e84aee5013c585d2efc as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/25caac8a2fc84e84aee5013c585d2efc 2024-12-05T11:14:56,525 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into 25caac8a2fc84e84aee5013c585d2efc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:56,525 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:56,525 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=13, startTime=1733397296418; duration=0sec 2024-12-05T11:14:56,525 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:56,525 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:14:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:56,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. as already flushing 2024-12-05T11:14:56,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733397296852, max timestamp=1733397296869Num uniques 859; 2024-12-05T11:14:56,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733397296852, max timestamp=1733397296869Num uniques 859; 2024-12-05T11:14:56,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T11:14:56,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733397296852, max timestamp=1733397296869Num uniques 859; 2024-12-05T11:14:56,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T11:14:56,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T11:14:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:56,909 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b8901cdd0cf0474cb569f694b80d5491_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b8901cdd0cf0474cb569f694b80d5491_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/40e3057dc88048349d46652f1d48fa6d, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:56,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/40e3057dc88048349d46652f1d48fa6d is 175, key is test_row_0/A:col10/1733397294744/Put/seqid=0 2024-12-05T11:14:56,911 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a 2024-12-05T11:14:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742027_1203 (size=31255) 2024-12-05T11:14:56,917 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into ad0a4dcd2c0e4fbebcea003a4fcfd04a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:14:56,917 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:56,917 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=13, startTime=1733397296418; duration=0sec 2024-12-05T11:14:56,917 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:14:56,917 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:14:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397356925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:56,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36020 deadline: 1733397356928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:56,933 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in b6afdaa222643745b7f25a1a31a4fff2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397357038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:57,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397357248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:57,315 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/40e3057dc88048349d46652f1d48fa6d 2024-12-05T11:14:57,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/dff2ee2d197b4b8a95581f3e7d665731 is 50, key is test_row_0/B:col10/1733397294744/Put/seqid=0 2024-12-05T11:14:57,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742028_1204 (size=12301) 2024-12-05T11:14:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397357558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:57,728 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/dff2ee2d197b4b8a95581f3e7d665731 2024-12-05T11:14:57,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/ef75f22bc44c49eb964f2c5050746faa is 50, key is test_row_0/C:col10/1733397294744/Put/seqid=0 2024-12-05T11:14:57,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742029_1205 (size=12301) 2024-12-05T11:14:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:14:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36040 deadline: 1733397358068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:14:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:14:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:14:58,140 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/ef75f22bc44c49eb964f2c5050746faa 2024-12-05T11:14:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/40e3057dc88048349d46652f1d48fa6d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d 2024-12-05T11:14:58,202 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d, entries=150, sequenceid=417, filesize=30.5 K 2024-12-05T11:14:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/dff2ee2d197b4b8a95581f3e7d665731 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/dff2ee2d197b4b8a95581f3e7d665731 2024-12-05T11:14:58,207 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/dff2ee2d197b4b8a95581f3e7d665731, entries=150, sequenceid=417, filesize=12.0 K 2024-12-05T11:14:58,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/ef75f22bc44c49eb964f2c5050746faa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/ef75f22bc44c49eb964f2c5050746faa 2024-12-05T11:14:58,212 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/ef75f22bc44c49eb964f2c5050746faa, entries=150, sequenceid=417, filesize=12.0 K 2024-12-05T11:14:58,213 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b6afdaa222643745b7f25a1a31a4fff2 in 1762ms, sequenceid=417, compaction requested=false 2024-12-05T11:14:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:14:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:14:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-05T11:14:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-05T11:14:58,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-05T11:14:58,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6840 sec 2024-12-05T11:14:58,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.6880 sec 2024-12-05T11:14:58,736 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-05T11:14:58,736 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:58,736 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,737 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:58,737 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-05T11:14:58,737 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:58,737 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,737 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:58,740 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-05T11:14:58,740 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:58,740 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,740 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:58,743 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-12-05T11:14:58,743 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:58,744 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,744 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:58,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-05T11:14:58,964 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-05T11:14:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:14:58,964 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:58,964 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:14:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:58,964 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:14:58,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:58,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:14:58,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205620133d1621044a5a42a94f2eb30ab86_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397298960/Put/seqid=0 2024-12-05T11:14:58,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742030_1206 (size=12454) 2024-12-05T11:14:59,082 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-12-05T11:14:59,082 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:14:59,082 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:59,082 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:14:59,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:14:59,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:14:59,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205620133d1621044a5a42a94f2eb30ab86_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205620133d1621044a5a42a94f2eb30ab86_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:14:59,388 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/01f596282693460496f71c8e79649e06, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:14:59,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/01f596282693460496f71c8e79649e06 is 175, key is test_row_0/A:col10/1733397298960/Put/seqid=0 2024-12-05T11:14:59,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742031_1207 (size=31255) 2024-12-05T11:14:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T11:14:59,699 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:14:59,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/01f596282693460496f71c8e79649e06 2024-12-05T11:14:59,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/0d16b8acdc0a4b42a029df3385b473e2 is 50, key is test_row_0/B:col10/1733397298960/Put/seqid=0 2024-12-05T11:14:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742032_1208 (size=12301) 2024-12-05T11:15:00,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/0d16b8acdc0a4b42a029df3385b473e2 2024-12-05T11:15:00,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/0e7fdc60a37f460dab35fff92ad94673 is 50, key is test_row_0/C:col10/1733397298960/Put/seqid=0 2024-12-05T11:15:00,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742033_1209 (size=12301) 2024-12-05T11:15:00,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/0e7fdc60a37f460dab35fff92ad94673 2024-12-05T11:15:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/01f596282693460496f71c8e79649e06 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06 2024-12-05T11:15:00,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06, entries=150, sequenceid=456, filesize=30.5 K 2024-12-05T11:15:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/0d16b8acdc0a4b42a029df3385b473e2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/0d16b8acdc0a4b42a029df3385b473e2 2024-12-05T11:15:00,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/0d16b8acdc0a4b42a029df3385b473e2, entries=150, sequenceid=456, filesize=12.0 K 2024-12-05T11:15:00,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/0e7fdc60a37f460dab35fff92ad94673 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/0e7fdc60a37f460dab35fff92ad94673 2024-12-05T11:15:00,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/0e7fdc60a37f460dab35fff92ad94673, entries=150, sequenceid=456, filesize=12.0 K 2024-12-05T11:15:00,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=6.71 KB/6870 for b6afdaa222643745b7f25a1a31a4fff2 in 1690ms, sequenceid=456, compaction requested=true 2024-12-05T11:15:00,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:15:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:00,654 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6afdaa222643745b7f25a1a31a4fff2:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:00,654 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/B is initiating minor compaction (all files) 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/A is initiating minor compaction (all files) 2024-12-05T11:15:00,655 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/B in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:00,655 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/A in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:00,655 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/35d48fec19ca47b79e043d39c25a28ed, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/dff2ee2d197b4b8a95581f3e7d665731, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/0d16b8acdc0a4b42a029df3385b473e2] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.7 K 2024-12-05T11:15:00,655 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=92.2 K 2024-12-05T11:15:00,655 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06] 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d48fec19ca47b79e043d39c25a28ed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733397294665 2024-12-05T11:15:00,655 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad0a4dcd2c0e4fbebcea003a4fcfd04a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733397294665 2024-12-05T11:15:00,656 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e3057dc88048349d46652f1d48fa6d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733397294744 2024-12-05T11:15:00,656 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting dff2ee2d197b4b8a95581f3e7d665731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733397294744 2024-12-05T11:15:00,656 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f596282693460496f71c8e79649e06, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733397296921 2024-12-05T11:15:00,656 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d16b8acdc0a4b42a029df3385b473e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733397296921 2024-12-05T11:15:00,663 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:15:00,664 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:00,664 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5742c683247a4790b3b0859b66c56453 is 50, key is test_row_0/B:col10/1733397298960/Put/seqid=0 2024-12-05T11:15:00,665 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053ef4d0170c4a4683b6d053d6bcfc6e0f_b6afdaa222643745b7f25a1a31a4fff2 store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:15:00,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742034_1210 (size=13085) 2024-12-05T11:15:00,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053ef4d0170c4a4683b6d053d6bcfc6e0f_b6afdaa222643745b7f25a1a31a4fff2, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:15:00,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053ef4d0170c4a4683b6d053d6bcfc6e0f_b6afdaa222643745b7f25a1a31a4fff2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:15:00,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742035_1211 (size=4469) 2024-12-05T11:15:01,080 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/5742c683247a4790b3b0859b66c56453 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5742c683247a4790b3b0859b66c56453 2024-12-05T11:15:01,081 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:01,081 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ba989896c2942ddab466d7332d637d2 is 175, key is test_row_0/A:col10/1733397298960/Put/seqid=0 2024-12-05T11:15:01,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742036_1212 (size=32039) 2024-12-05T11:15:01,085 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/B of b6afdaa222643745b7f25a1a31a4fff2 into 5742c683247a4790b3b0859b66c56453(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:01,085 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:15:01,085 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/B, priority=13, startTime=1733397300654; duration=0sec 2024-12-05T11:15:01,085 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:01,085 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:B 2024-12-05T11:15:01,085 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:01,086 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:01,086 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): b6afdaa222643745b7f25a1a31a4fff2/C is initiating minor compaction (all files) 2024-12-05T11:15:01,086 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6afdaa222643745b7f25a1a31a4fff2/C in TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:01,086 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/25caac8a2fc84e84aee5013c585d2efc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/ef75f22bc44c49eb964f2c5050746faa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/0e7fdc60a37f460dab35fff92ad94673] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp, totalSize=36.7 K 2024-12-05T11:15:01,087 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 25caac8a2fc84e84aee5013c585d2efc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733397294665 2024-12-05T11:15:01,087 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ef75f22bc44c49eb964f2c5050746faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733397294744 2024-12-05T11:15:01,087 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7fdc60a37f460dab35fff92ad94673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733397296921 2024-12-05T11:15:01,094 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6afdaa222643745b7f25a1a31a4fff2#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:01,094 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/93492d957ba04d97897f0260a6c105e0 is 50, key is test_row_0/C:col10/1733397298960/Put/seqid=0 2024-12-05T11:15:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742037_1213 (size=13085) 2024-12-05T11:15:01,495 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/2ba989896c2942ddab466d7332d637d2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ba989896c2942ddab466d7332d637d2 2024-12-05T11:15:01,503 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/A of b6afdaa222643745b7f25a1a31a4fff2 into 2ba989896c2942ddab466d7332d637d2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:01,503 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:15:01,503 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/A, priority=13, startTime=1733397300653; duration=0sec 2024-12-05T11:15:01,503 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:01,503 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:A 2024-12-05T11:15:01,504 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/93492d957ba04d97897f0260a6c105e0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/93492d957ba04d97897f0260a6c105e0 2024-12-05T11:15:01,509 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6afdaa222643745b7f25a1a31a4fff2/C of b6afdaa222643745b7f25a1a31a4fff2 into 93492d957ba04d97897f0260a6c105e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:01,509 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6afdaa222643745b7f25a1a31a4fff2: 2024-12-05T11:15:01,509 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., storeName=b6afdaa222643745b7f25a1a31a4fff2/C, priority=13, startTime=1733397300654; duration=0sec 2024-12-05T11:15:01,509 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:01,509 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6afdaa222643745b7f25a1a31a4fff2:C 2024-12-05T11:15:02,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:15:02,581 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-05T11:15:02,581 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:02,581 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:02,581 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:02,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:05,662 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-12-05T11:15:05,662 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:05,662 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:05,662 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:05,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:05,723 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-05T11:15:05,723 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:05,723 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:05,723 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:05,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4435 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4243 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1895 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5681 rows 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1917 2024-12-05T11:15:05,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5747 rows 2024-12-05T11:15:05,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:15:05,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:15:05,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:05,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:05,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:05,727 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:15:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:15:05,732 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397305731"}]},"ts":"1733397305731"} 2024-12-05T11:15:05,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:15:05,733 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:15:05,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:15:05,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, UNASSIGN}] 2024-12-05T11:15:05,735 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, UNASSIGN 2024-12-05T11:15:05,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:05,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, UNASSIGN because future has completed 2024-12-05T11:15:05,737 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:15:05,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:15:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:15:05,891 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:05,891 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:15:05,891 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing b6afdaa222643745b7f25a1a31a4fff2, disabling compactions & flushes 2024-12-05T11:15:05,892 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:05,892 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:05,892 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. after waiting 0 ms 2024-12-05T11:15:05,892 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:05,892 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing b6afdaa222643745b7f25a1a31a4fff2 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T11:15:05,893 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=A 2024-12-05T11:15:05,893 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:05,893 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=B 2024-12-05T11:15:05,894 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:05,894 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b6afdaa222643745b7f25a1a31a4fff2, store=C 2024-12-05T11:15:05,894 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:05,903 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b24d1113e7b3441c9583ee0119520237_b6afdaa222643745b7f25a1a31a4fff2 is 50, key is test_row_0/A:col10/1733397305720/Put/seqid=0 2024-12-05T11:15:05,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742038_1214 (size=12454) 2024-12-05T11:15:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:15:06,307 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:06,313 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b24d1113e7b3441c9583ee0119520237_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b24d1113e7b3441c9583ee0119520237_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:06,314 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/57a78db532bc4912afd7329fda641d1f, store: [table=TestAcidGuarantees family=A region=b6afdaa222643745b7f25a1a31a4fff2] 2024-12-05T11:15:06,315 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/57a78db532bc4912afd7329fda641d1f is 175, key is test_row_0/A:col10/1733397305720/Put/seqid=0 2024-12-05T11:15:06,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742039_1215 (size=31255) 2024-12-05T11:15:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:15:06,720 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=466, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/57a78db532bc4912afd7329fda641d1f 2024-12-05T11:15:06,734 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/b2fa7163bbe54e7ea397e21d1ab4504c is 50, key is test_row_0/B:col10/1733397305720/Put/seqid=0 2024-12-05T11:15:06,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742040_1216 (size=12301) 2024-12-05T11:15:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:15:07,138 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/b2fa7163bbe54e7ea397e21d1ab4504c 2024-12-05T11:15:07,146 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/49bdc2fa730d48f5bc1dacc48e573d4d is 50, key is test_row_0/C:col10/1733397305720/Put/seqid=0 2024-12-05T11:15:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742041_1217 (size=12301) 2024-12-05T11:15:07,551 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/49bdc2fa730d48f5bc1dacc48e573d4d 2024-12-05T11:15:07,563 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/A/57a78db532bc4912afd7329fda641d1f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/57a78db532bc4912afd7329fda641d1f 2024-12-05T11:15:07,568 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/57a78db532bc4912afd7329fda641d1f, entries=150, sequenceid=466, filesize=30.5 K 2024-12-05T11:15:07,568 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/B/b2fa7163bbe54e7ea397e21d1ab4504c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b2fa7163bbe54e7ea397e21d1ab4504c 2024-12-05T11:15:07,573 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b2fa7163bbe54e7ea397e21d1ab4504c, entries=150, sequenceid=466, filesize=12.0 K 2024-12-05T11:15:07,574 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/.tmp/C/49bdc2fa730d48f5bc1dacc48e573d4d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/49bdc2fa730d48f5bc1dacc48e573d4d 2024-12-05T11:15:07,578 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/49bdc2fa730d48f5bc1dacc48e573d4d, entries=150, sequenceid=466, filesize=12.0 K 2024-12-05T11:15:07,579 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b6afdaa222643745b7f25a1a31a4fff2 in 1687ms, sequenceid=466, compaction requested=false 2024-12-05T11:15:07,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cdcae28cd6849f1b2257c0584b4617a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/0c3d438e9c08402093591ef56f34e892, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1c0023e36f9f4d4fa0f776143bddf53f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e012ced140564f78ba655205f91e6c76, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/050936b167d64be9b1dc66bf02657651, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06] to archive 2024-12-05T11:15:07,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:15:07,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/512ae78cc32240ecb033954bba0daf0f 2024-12-05T11:15:07,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/761e641c19ce4048ad95816a5279b73a 2024-12-05T11:15:07,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7b0229c076e84afe87712771703e0376 2024-12-05T11:15:07,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e84d411a0a1a46d999fe69616e26e2f2 2024-12-05T11:15:07,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cdcae28cd6849f1b2257c0584b4617a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cdcae28cd6849f1b2257c0584b4617a 2024-12-05T11:15:07,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/7cc93344d5b3495ca675cd29963b2bc8 2024-12-05T11:15:07,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f55d5add319449ce850f20e12ef41282 2024-12-05T11:15:07,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2a9c236ed2834ded95948c5aa50274db 2024-12-05T11:15:07,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/0c3d438e9c08402093591ef56f34e892 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/0c3d438e9c08402093591ef56f34e892 2024-12-05T11:15:07,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ec7efef26a647a2929b06d673a2af9e 2024-12-05T11:15:07,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1451fac580f2473891119bf138c537f3 2024-12-05T11:15:07,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1c0023e36f9f4d4fa0f776143bddf53f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/1c0023e36f9f4d4fa0f776143bddf53f 2024-12-05T11:15:07,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/41521f8a211d48f28bfbd6fab61a4ab6 2024-12-05T11:15:07,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2b684782e94949a7b3226d02c67a0589 2024-12-05T11:15:07,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e012ced140564f78ba655205f91e6c76 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e012ced140564f78ba655205f91e6c76 2024-12-05T11:15:07,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/12b87c3b920d47348e0e71a76fcbd387 2024-12-05T11:15:07,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ae5a5da7f104da2b719550f1285b251 2024-12-05T11:15:07,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/050936b167d64be9b1dc66bf02657651 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/050936b167d64be9b1dc66bf02657651 2024-12-05T11:15:07,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/f5ad65955eb2417b909e0b6c56593807 2024-12-05T11:15:07,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/e5bb4e4129674616b140845bb08f3fbc 2024-12-05T11:15:07,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/ad0a4dcd2c0e4fbebcea003a4fcfd04a 2024-12-05T11:15:07,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/40e3057dc88048349d46652f1d48fa6d 2024-12-05T11:15:07,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/01f596282693460496f71c8e79649e06 2024-12-05T11:15:07,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c07b4e35e0a34922bf06b8ca9af882c2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/eb42ca6c15a94d55b70b6b5017742bf3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/734206cc2fd64da891acb6eab2868fa0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/4f48db40350d4883bccf24466f3a2fc7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/3d651d5fc81f46498633762782c66bfc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c385b3a75d86435bb7c0668e9e912c41, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/9026b4874602490abe065185b37a4e95, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c532b6f463dd44dfa4fee8fad3cb9f23, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/cb3df922288243fdbad871435b66d05b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b52136e7b51346c5985ad4cffe23f1b2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/e4fbe82db38942f4be939dafd10f90e7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/45e5099dab954e42a2a36683efd63701, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/1658a6072aa047659b623ad6e77a4c66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/265a9598a4cf4aa78b403cc53a01473b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5e25854f53334007b155b2f4c6bd88b8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/46e7d93cc08647e8a275eb92adf597f7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/7b03e49d3b2f4623ab32203ee4c2dc0e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/2a8a60a422e8459b8685b9c34f55733b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/35d48fec19ca47b79e043d39c25a28ed, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/06023d28cad743dfa873c038ff323206, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/dff2ee2d197b4b8a95581f3e7d665731, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/0d16b8acdc0a4b42a029df3385b473e2] to archive 2024-12-05T11:15:07,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:15:07,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c07b4e35e0a34922bf06b8ca9af882c2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c07b4e35e0a34922bf06b8ca9af882c2 2024-12-05T11:15:07,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/eb42ca6c15a94d55b70b6b5017742bf3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/eb42ca6c15a94d55b70b6b5017742bf3 2024-12-05T11:15:07,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/734206cc2fd64da891acb6eab2868fa0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/734206cc2fd64da891acb6eab2868fa0 2024-12-05T11:15:07,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/4f48db40350d4883bccf24466f3a2fc7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/4f48db40350d4883bccf24466f3a2fc7 2024-12-05T11:15:07,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/3d651d5fc81f46498633762782c66bfc to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/3d651d5fc81f46498633762782c66bfc 2024-12-05T11:15:07,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c385b3a75d86435bb7c0668e9e912c41 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c385b3a75d86435bb7c0668e9e912c41 2024-12-05T11:15:07,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/9026b4874602490abe065185b37a4e95 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/9026b4874602490abe065185b37a4e95 2024-12-05T11:15:07,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c532b6f463dd44dfa4fee8fad3cb9f23 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/c532b6f463dd44dfa4fee8fad3cb9f23 2024-12-05T11:15:07,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/cb3df922288243fdbad871435b66d05b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/cb3df922288243fdbad871435b66d05b 2024-12-05T11:15:07,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b52136e7b51346c5985ad4cffe23f1b2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b52136e7b51346c5985ad4cffe23f1b2 2024-12-05T11:15:07,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5c2c9bf6b36c4a6b8b2c60d4d4a4edbb 2024-12-05T11:15:07,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/e4fbe82db38942f4be939dafd10f90e7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/e4fbe82db38942f4be939dafd10f90e7 2024-12-05T11:15:07,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/45e5099dab954e42a2a36683efd63701 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/45e5099dab954e42a2a36683efd63701 2024-12-05T11:15:07,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/1658a6072aa047659b623ad6e77a4c66 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/1658a6072aa047659b623ad6e77a4c66 2024-12-05T11:15:07,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/265a9598a4cf4aa78b403cc53a01473b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/265a9598a4cf4aa78b403cc53a01473b 2024-12-05T11:15:07,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5e25854f53334007b155b2f4c6bd88b8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5e25854f53334007b155b2f4c6bd88b8 2024-12-05T11:15:07,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/46e7d93cc08647e8a275eb92adf597f7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/46e7d93cc08647e8a275eb92adf597f7 2024-12-05T11:15:07,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/7b03e49d3b2f4623ab32203ee4c2dc0e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/7b03e49d3b2f4623ab32203ee4c2dc0e 2024-12-05T11:15:07,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/2a8a60a422e8459b8685b9c34f55733b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/2a8a60a422e8459b8685b9c34f55733b 2024-12-05T11:15:07,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/35d48fec19ca47b79e043d39c25a28ed to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/35d48fec19ca47b79e043d39c25a28ed 2024-12-05T11:15:07,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/06023d28cad743dfa873c038ff323206 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/06023d28cad743dfa873c038ff323206 2024-12-05T11:15:07,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/dff2ee2d197b4b8a95581f3e7d665731 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/dff2ee2d197b4b8a95581f3e7d665731 2024-12-05T11:15:07,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/0d16b8acdc0a4b42a029df3385b473e2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/0d16b8acdc0a4b42a029df3385b473e2 2024-12-05T11:15:07,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3922b2e3a73c49748e11adb1bcdc4fbb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3bd48046e733446a8a921214b53db9bc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/83ee3f9e2aba4f87a7ec7adfced5efb6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c3f511bbcdeb4ec29ae0ad8446467bbd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c20be753a1cb48a0b7835f591a0d0216, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/52e5f5a01503460583f68d20b33ab0a9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a209a6c73de341e29d1e7f3d3509e43a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/aa5ac87933dc4212aebd6b13665a6aa0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d264740d91ad42bdbc76cb3c43a9f7b2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/2627e6a3247144a68b5781ad80875bad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3b56fb3499d1470b8a4d91034aaac7bf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/75fa4f7b2f46425586aa4d0106684fbf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/80a5e1535a49413bb43e13a9956af0c6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/12d19d33b9bc4955beddc19251acd447, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/6f21271c4b234b98b582c47d72917e8c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c624c8e80285441eb730f82eea8f81aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/fa4e19fdf24f40ac8b4fe4985a3de43d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a8a0bf8ef5224c75be19e878f8cbb676, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/04af4ea17706485db24f1e8b5d7e0460, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/25caac8a2fc84e84aee5013c585d2efc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d19f736682444ff9a07a1537122f93ec, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/ef75f22bc44c49eb964f2c5050746faa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/0e7fdc60a37f460dab35fff92ad94673] to archive 2024-12-05T11:15:07,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:15:07,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3922b2e3a73c49748e11adb1bcdc4fbb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3922b2e3a73c49748e11adb1bcdc4fbb 2024-12-05T11:15:07,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3bd48046e733446a8a921214b53db9bc to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3bd48046e733446a8a921214b53db9bc 2024-12-05T11:15:07,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/83ee3f9e2aba4f87a7ec7adfced5efb6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/83ee3f9e2aba4f87a7ec7adfced5efb6 2024-12-05T11:15:07,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c3f511bbcdeb4ec29ae0ad8446467bbd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c3f511bbcdeb4ec29ae0ad8446467bbd 2024-12-05T11:15:07,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c20be753a1cb48a0b7835f591a0d0216 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c20be753a1cb48a0b7835f591a0d0216 2024-12-05T11:15:07,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/52e5f5a01503460583f68d20b33ab0a9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/52e5f5a01503460583f68d20b33ab0a9 2024-12-05T11:15:07,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a209a6c73de341e29d1e7f3d3509e43a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a209a6c73de341e29d1e7f3d3509e43a 2024-12-05T11:15:07,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/aa5ac87933dc4212aebd6b13665a6aa0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/aa5ac87933dc4212aebd6b13665a6aa0 2024-12-05T11:15:07,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d264740d91ad42bdbc76cb3c43a9f7b2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d264740d91ad42bdbc76cb3c43a9f7b2 2024-12-05T11:15:07,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/2627e6a3247144a68b5781ad80875bad to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/2627e6a3247144a68b5781ad80875bad 2024-12-05T11:15:07,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3b56fb3499d1470b8a4d91034aaac7bf to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/3b56fb3499d1470b8a4d91034aaac7bf 2024-12-05T11:15:07,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/75fa4f7b2f46425586aa4d0106684fbf to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/75fa4f7b2f46425586aa4d0106684fbf 2024-12-05T11:15:07,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/80a5e1535a49413bb43e13a9956af0c6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/80a5e1535a49413bb43e13a9956af0c6 2024-12-05T11:15:07,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/12d19d33b9bc4955beddc19251acd447 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/12d19d33b9bc4955beddc19251acd447 2024-12-05T11:15:07,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/6f21271c4b234b98b582c47d72917e8c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/6f21271c4b234b98b582c47d72917e8c 2024-12-05T11:15:07,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c624c8e80285441eb730f82eea8f81aa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/c624c8e80285441eb730f82eea8f81aa 2024-12-05T11:15:07,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/fa4e19fdf24f40ac8b4fe4985a3de43d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/fa4e19fdf24f40ac8b4fe4985a3de43d 2024-12-05T11:15:07,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a8a0bf8ef5224c75be19e878f8cbb676 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/a8a0bf8ef5224c75be19e878f8cbb676 2024-12-05T11:15:07,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/04af4ea17706485db24f1e8b5d7e0460 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/04af4ea17706485db24f1e8b5d7e0460 2024-12-05T11:15:07,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/25caac8a2fc84e84aee5013c585d2efc to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/25caac8a2fc84e84aee5013c585d2efc 2024-12-05T11:15:07,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d19f736682444ff9a07a1537122f93ec to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/d19f736682444ff9a07a1537122f93ec 2024-12-05T11:15:07,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/ef75f22bc44c49eb964f2c5050746faa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/ef75f22bc44c49eb964f2c5050746faa 2024-12-05T11:15:07,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/0e7fdc60a37f460dab35fff92ad94673 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/0e7fdc60a37f460dab35fff92ad94673 2024-12-05T11:15:07,657 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/recovered.edits/469.seqid, newMaxSeqId=469, maxSeqId=4 2024-12-05T11:15:07,658 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. 2024-12-05T11:15:07,658 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for b6afdaa222643745b7f25a1a31a4fff2: Waiting for close lock at 1733397305891Running coprocessor pre-close hooks at 1733397305891Disabling compacts and flushes for region at 1733397305891Disabling writes for close at 1733397305892 (+1 ms)Obtaining lock to block concurrent updates at 1733397305892Preparing flush snapshotting stores in b6afdaa222643745b7f25a1a31a4fff2 at 1733397305892Finished memstore snapshotting TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733397305894 (+2 ms)Flushing stores of TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2. at 1733397305895 (+1 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/A: creating writer at 1733397305895Flushing mob file b6afdaa222643745b7f25a1a31a4fff2/A: appending metadata at 1733397305903 (+8 ms)Flushing mob file b6afdaa222643745b7f25a1a31a4fff2/A: closing flushed file at 1733397305903Flushing b6afdaa222643745b7f25a1a31a4fff2/A: appending metadata at 1733397306314 (+411 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/A: closing flushed file at 1733397306314Flushing b6afdaa222643745b7f25a1a31a4fff2/B: creating writer at 1733397306731 (+417 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/B: appending metadata at 1733397306733 (+2 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/B: closing flushed file at 1733397306734 (+1 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/C: creating writer at 1733397307143 (+409 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/C: appending metadata at 1733397307145 (+2 ms)Flushing b6afdaa222643745b7f25a1a31a4fff2/C: closing flushed file at 1733397307145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f2fb8d: reopening flushed file at 1733397307562 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c7bbb9: reopening flushed file at 1733397307568 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dbda833: reopening flushed file at 1733397307573 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b6afdaa222643745b7f25a1a31a4fff2 in 1687ms, sequenceid=466, compaction requested=false at 1733397307579 (+6 ms)Writing region close event to WAL at 1733397307654 (+75 ms)Running coprocessor post-close hooks at 1733397307657 (+3 ms)Closed at 1733397307658 (+1 ms) 2024-12-05T11:15:07,660 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,660 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=b6afdaa222643745b7f25a1a31a4fff2, regionState=CLOSED 2024-12-05T11:15:07,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:15:07,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T11:15:07,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure b6afdaa222643745b7f25a1a31a4fff2, server=6706a97ae65f,42931,1733397243639 in 1.9250 sec 2024-12-05T11:15:07,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-12-05T11:15:07,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b6afdaa222643745b7f25a1a31a4fff2, UNASSIGN in 1.9280 sec 2024-12-05T11:15:07,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T11:15:07,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9320 sec 2024-12-05T11:15:07,668 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397307668"}]},"ts":"1733397307668"} 2024-12-05T11:15:07,669 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:15:07,669 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:15:07,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9420 sec 2024-12-05T11:15:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T11:15:07,890 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:07,891 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:15:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:07,895 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:15:07,898 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:07,902 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,907 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/recovered.edits] 2024-12-05T11:15:07,913 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ba989896c2942ddab466d7332d637d2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/2ba989896c2942ddab466d7332d637d2 2024-12-05T11:15:07,916 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/57a78db532bc4912afd7329fda641d1f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/A/57a78db532bc4912afd7329fda641d1f 2024-12-05T11:15:07,921 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5742c683247a4790b3b0859b66c56453 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/5742c683247a4790b3b0859b66c56453 2024-12-05T11:15:07,923 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b2fa7163bbe54e7ea397e21d1ab4504c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/B/b2fa7163bbe54e7ea397e21d1ab4504c 2024-12-05T11:15:07,927 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/49bdc2fa730d48f5bc1dacc48e573d4d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/49bdc2fa730d48f5bc1dacc48e573d4d 2024-12-05T11:15:07,929 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/93492d957ba04d97897f0260a6c105e0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/C/93492d957ba04d97897f0260a6c105e0 2024-12-05T11:15:07,933 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/recovered.edits/469.seqid to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2/recovered.edits/469.seqid 2024-12-05T11:15:07,934 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,934 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:15:07,934 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:15:07,935 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T11:15:07,941 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205122dcd22de2a4d4e892fdbe5822bce0a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205122dcd22de2a4d4e892fdbe5822bce0a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,942 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120516dfc22db1dd4fd9a2a543868c01e53e_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120516dfc22db1dd4fd9a2a543868c01e53e_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,943 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205279443abe10449b680674d98e2997b5d_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205279443abe10449b680674d98e2997b5d_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,945 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053905430c2e204ef6824f54d3cc5d7d60_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053905430c2e204ef6824f54d3cc5d7d60_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,946 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053d196c7d3ec649d682660617658df76b_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053d196c7d3ec649d682660617658df76b_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,947 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120558b938fd5b1a49ccab340c6f2b49806a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120558b938fd5b1a49ccab340c6f2b49806a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,947 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205620133d1621044a5a42a94f2eb30ab86_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205620133d1621044a5a42a94f2eb30ab86_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,949 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056c283e51af914fe7a2b3c84b97996597_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056c283e51af914fe7a2b3c84b97996597_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,950 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205741925c6f02a44318fdc790d55229833_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205741925c6f02a44318fdc790d55229833_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,951 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058b685be3f35a4f7ba2463b7907d7ce72_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058b685be3f35a4f7ba2463b7907d7ce72_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,952 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205abad732ea0aa419ead3da07cf4223c9b_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205abad732ea0aa419ead3da07cf4223c9b_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,953 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ac22cc10352c450ca2a4b8a4c730f6bd_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ac22cc10352c450ca2a4b8a4c730f6bd_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,954 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205af2e8c43c6e5424b9b90aca1d1a8ff12_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205af2e8c43c6e5424b9b90aca1d1a8ff12_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,955 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b24d1113e7b3441c9583ee0119520237_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b24d1113e7b3441c9583ee0119520237_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,956 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b8901cdd0cf0474cb569f694b80d5491_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b8901cdd0cf0474cb569f694b80d5491_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,957 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c21987b1c82f44079f7de7805bce7c6f_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c21987b1c82f44079f7de7805bce7c6f_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,958 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e38e701f1884430db99a82546d75c11a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e38e701f1884430db99a82546d75c11a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,958 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fcc1bc63b07f453dad7d06f39e5f3f7a_b6afdaa222643745b7f25a1a31a4fff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fcc1bc63b07f453dad7d06f39e5f3f7a_b6afdaa222643745b7f25a1a31a4fff2 2024-12-05T11:15:07,959 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:15:07,961 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:07,963 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:15:07,964 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:15:07,965 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:07,965 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:15:07,965 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733397307965"}]},"ts":"9223372036854775807"} 2024-12-05T11:15:07,967 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:15:07,967 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b6afdaa222643745b7f25a1a31a4fff2, NAME => 'TestAcidGuarantees,,1733397275489.b6afdaa222643745b7f25a1a31a4fff2.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:15:07,967 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:15:07,967 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733397307967"}]},"ts":"9223372036854775807"} 2024-12-05T11:15:07,969 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:15:07,969 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:07,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-12-05T11:15:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T11:15:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:15:08,009 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:08,017 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 369), ProcessCount=11 (was 11), AvailableMemoryMB=5982 (was 6184) 2024-12-05T11:15:08,024 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=5982 2024-12-05T11:15:08,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:15:08,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:15:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:08,027 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:15:08,027 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:08,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-12-05T11:15:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-05T11:15:08,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:15:08,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742042_1218 (size=1061) 2024-12-05T11:15:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-05T11:15:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-05T11:15:08,451 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:15:08,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742043_1219 (size=53) 2024-12-05T11:15:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-05T11:15:08,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:15:08,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 04e661eeba0fa942008ec581fcd28e79, disabling compactions & flushes 2024-12-05T11:15:08,863 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:08,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:08,864 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. after waiting 1 ms 2024-12-05T11:15:08,864 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:08,864 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:08,864 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04e661eeba0fa942008ec581fcd28e79: Waiting for close lock at 1733397308863Disabling compacts and flushes for region at 1733397308863Disabling writes for close at 1733397308864 (+1 ms)Writing region close event to WAL at 1733397308864Closed at 1733397308864 2024-12-05T11:15:08,867 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:15:08,868 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733397308867"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733397308867"}]},"ts":"1733397308867"} 2024-12-05T11:15:08,870 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:15:08,871 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:15:08,871 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397308871"}]},"ts":"1733397308871"} 2024-12-05T11:15:08,873 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:15:08,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, ASSIGN}] 2024-12-05T11:15:08,874 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, ASSIGN 2024-12-05T11:15:08,875 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, ASSIGN; state=OFFLINE, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=false 2024-12-05T11:15:09,026 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=04e661eeba0fa942008ec581fcd28e79, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:09,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, ASSIGN because future has completed 2024-12-05T11:15:09,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:15:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-05T11:15:09,189 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:09,189 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:15:09,190 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,190 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:15:09,190 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,191 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,193 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,195 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:09,196 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04e661eeba0fa942008ec581fcd28e79 columnFamilyName A 2024-12-05T11:15:09,196 DEBUG [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:09,197 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.HStore(327): Store=04e661eeba0fa942008ec581fcd28e79/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:09,197 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,199 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:09,199 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04e661eeba0fa942008ec581fcd28e79 columnFamilyName B 2024-12-05T11:15:09,200 DEBUG [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:09,200 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.HStore(327): Store=04e661eeba0fa942008ec581fcd28e79/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:09,200 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,201 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:09,201 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04e661eeba0fa942008ec581fcd28e79 columnFamilyName C 2024-12-05T11:15:09,201 DEBUG [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:09,202 INFO [StoreOpener-04e661eeba0fa942008ec581fcd28e79-1 {}] regionserver.HStore(327): Store=04e661eeba0fa942008ec581fcd28e79/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:09,202 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:09,202 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,203 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,203 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,204 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,204 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,204 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:15:09,206 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,207 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:15:09,208 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 04e661eeba0fa942008ec581fcd28e79; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63168996, jitterRate=-0.05870860815048218}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:15:09,208 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:09,208 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 04e661eeba0fa942008ec581fcd28e79: Running coprocessor pre-open hook at 1733397309191Writing region info on filesystem at 1733397309191Initializing all the Stores at 1733397309192 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397309192Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397309193 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397309193Cleaning up temporary data from old regions at 1733397309204 (+11 ms)Running coprocessor post-open hooks at 1733397309208 (+4 ms)Region opened successfully at 1733397309208 2024-12-05T11:15:09,209 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., pid=55, masterSystemTime=1733397309183 2024-12-05T11:15:09,211 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:09,211 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:09,211 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=04e661eeba0fa942008ec581fcd28e79, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:09,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:15:09,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-05T11:15:09,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure 04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 in 183 msec 2024-12-05T11:15:09,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-05T11:15:09,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, ASSIGN in 343 msec 2024-12-05T11:15:09,219 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:15:09,219 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397309219"}]},"ts":"1733397309219"} 2024-12-05T11:15:09,221 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:15:09,222 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:15:09,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1960 sec 2024-12-05T11:15:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-05T11:15:10,180 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:10,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6d9a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,186 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef5f108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d72d05c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,192 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c187f19 2024-12-05T11:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:15:10,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:15:10,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ddf112b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,199 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b81fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,201 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7a00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331dd6dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,203 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b2e17f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b14ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5ff9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,206 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26adfeb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8bb105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ed25b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7401c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,212 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310f0bda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4681136b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,215 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b1950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,216 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253b0883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3776ce94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,219 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e970b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,220 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81ee97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab80ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,222 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6d81a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13acb63e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662986c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,225 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,226 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a5f521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b02cb70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,228 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1680472c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,229 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ade7820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7d41b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:10,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:10,231 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7732c9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:10,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9751f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:10,233 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,233 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,233 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,233 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,233 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,234 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,234 DEBUG [Thread-1046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-05T11:15:10,235 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,235 DEBUG [Thread-1037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T11:15:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,236 DEBUG [Thread-1035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,237 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:10,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:10,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:10,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,241 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:15:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:10,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:15:10,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:10,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:10,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:10,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:10,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:10,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:10,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733397310257, max timestamp=1733397310263Num uniques 859; 2024-12-05T11:15:10,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T11:15:10,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397310257, max timestamp=1733397310263Num uniques 859; 2024-12-05T11:15:10,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T11:15:10,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397310257, max timestamp=1733397310263Num uniques 859; 2024-12-05T11:15:10,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T11:15:10,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/e576f13a197b48d0b4665b555d23f535 is 50, key is test_row_0/A:col10/1733397310245/Put/seqid=0 2024-12-05T11:15:10,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742044_1220 (size=9657) 2024-12-05T11:15:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397370279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397370280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397370280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397370281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397370282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T11:15:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397370389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397370389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397370389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397370389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397370389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T11:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T11:15:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397370598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397370598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397370598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397370598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397370598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/e576f13a197b48d0b4665b555d23f535 2024-12-05T11:15:10,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/11be532676324a039cca2577674245f1 is 50, key is test_row_0/B:col10/1733397310245/Put/seqid=0 2024-12-05T11:15:10,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T11:15:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742045_1221 (size=9657) 2024-12-05T11:15:10,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T11:15:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:10,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T11:15:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397370908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397370908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397370908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397370908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397370908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T11:15:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:11,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:11,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/11be532676324a039cca2577674245f1 2024-12-05T11:15:11,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/7d06e70ea99e4835af72b467c15b3dbd is 50, key is test_row_0/C:col10/1733397310245/Put/seqid=0 2024-12-05T11:15:11,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742046_1222 (size=9657) 2024-12-05T11:15:11,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/7d06e70ea99e4835af72b467c15b3dbd 2024-12-05T11:15:11,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/e576f13a197b48d0b4665b555d23f535 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/e576f13a197b48d0b4665b555d23f535 2024-12-05T11:15:11,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/e576f13a197b48d0b4665b555d23f535, entries=100, sequenceid=11, filesize=9.4 K 2024-12-05T11:15:11,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/11be532676324a039cca2577674245f1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/11be532676324a039cca2577674245f1 2024-12-05T11:15:11,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/11be532676324a039cca2577674245f1, entries=100, sequenceid=11, filesize=9.4 K 2024-12-05T11:15:11,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/7d06e70ea99e4835af72b467c15b3dbd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/7d06e70ea99e4835af72b467c15b3dbd 2024-12-05T11:15:11,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/7d06e70ea99e4835af72b467c15b3dbd, entries=100, sequenceid=11, filesize=9.4 K 2024-12-05T11:15:11,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.39 KB/136592 for 04e661eeba0fa942008ec581fcd28e79 in 920ms, sequenceid=11, compaction requested=false 2024-12-05T11:15:11,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-05T11:15:11,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:11,167 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=133.39 KB heapSize=378.47 KB 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:11,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8e666398a9314713b96fe31eb4ec3e3f is 50, key is test_row_0/A:col10/1733397310280/Put/seqid=0 2024-12-05T11:15:11,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742047_1223 (size=12001) 2024-12-05T11:15:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T11:15:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:11,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:11,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397371421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397371422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397371422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397371425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397371425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397371528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397371528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397371529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397371539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397371539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,576 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8e666398a9314713b96fe31eb4ec3e3f 2024-12-05T11:15:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/d202b45c691c499fbfeb364ce685cf5e is 50, key is test_row_0/B:col10/1733397310280/Put/seqid=0 2024-12-05T11:15:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742048_1224 (size=12001) 2024-12-05T11:15:11,587 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/d202b45c691c499fbfeb364ce685cf5e 2024-12-05T11:15:11,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/244150eef7ce4d81a854dc278a6c6133 is 50, key is test_row_0/C:col10/1733397310280/Put/seqid=0 2024-12-05T11:15:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742049_1225 (size=12001) 2024-12-05T11:15:11,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397371738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397371738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397371739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397371748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397371748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:11,999 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/244150eef7ce4d81a854dc278a6c6133 2024-12-05T11:15:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8e666398a9314713b96fe31eb4ec3e3f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8e666398a9314713b96fe31eb4ec3e3f 2024-12-05T11:15:12,008 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8e666398a9314713b96fe31eb4ec3e3f, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T11:15:12,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/d202b45c691c499fbfeb364ce685cf5e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d202b45c691c499fbfeb364ce685cf5e 2024-12-05T11:15:12,013 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d202b45c691c499fbfeb364ce685cf5e, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T11:15:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/244150eef7ce4d81a854dc278a6c6133 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/244150eef7ce4d81a854dc278a6c6133 2024-12-05T11:15:12,020 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/244150eef7ce4d81a854dc278a6c6133, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T11:15:12,021 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~133.39 KB/136592, heapSize ~378.42 KB/387504, currentSize=46.96 KB/48090 for 04e661eeba0fa942008ec581fcd28e79 in 854ms, sequenceid=47, compaction requested=false 2024-12-05T11:15:12,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:12,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-05T11:15:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-05T11:15:12,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-05T11:15:12,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-12-05T11:15:12,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.7910 sec 2024-12-05T11:15:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:12,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:15:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:12,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:12,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/11d6b5652a2a4d6caf07a66da2ee485b is 50, key is test_row_0/A:col10/1733397312053/Put/seqid=0 2024-12-05T11:15:12,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742050_1226 (size=12001) 2024-12-05T11:15:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397372071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397372074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397372075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397372076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397372077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397372178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397372188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397372188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397372189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397372189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T11:15:12,389 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:12,389 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-05T11:15:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:15:12,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,391 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397372388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:12,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397372398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397372398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397372399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397372399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/11d6b5652a2a4d6caf07a66da2ee485b 2024-12-05T11:15:12,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/a9cef8039dc945fb9fd961e14df99f6d is 50, key is test_row_0/B:col10/1733397312053/Put/seqid=0 2024-12-05T11:15:12,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742051_1227 (size=12001) 2024-12-05T11:15:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:15:12,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397372698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:15:12,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397372708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397372708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397372708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397372708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:12,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:12,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:12,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:12,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:12,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/a9cef8039dc945fb9fd961e14df99f6d 2024-12-05T11:15:12,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/4ffccbf60f4340da9d8be00a215c884e is 50, key is test_row_0/C:col10/1733397312053/Put/seqid=0 2024-12-05T11:15:12,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742052_1228 (size=12001) 2024-12-05T11:15:13,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:13,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:15:13,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:13,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:13,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,178 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:15:13,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397373208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:13,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397373218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:13,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397373219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397373219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:13,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397373219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:13,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/4ffccbf60f4340da9d8be00a215c884e 2024-12-05T11:15:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/11d6b5652a2a4d6caf07a66da2ee485b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/11d6b5652a2a4d6caf07a66da2ee485b 2024-12-05T11:15:13,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:13,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:13,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/11d6b5652a2a4d6caf07a66da2ee485b, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:15:13,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/a9cef8039dc945fb9fd961e14df99f6d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/a9cef8039dc945fb9fd961e14df99f6d 2024-12-05T11:15:13,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/a9cef8039dc945fb9fd961e14df99f6d, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:15:13,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/4ffccbf60f4340da9d8be00a215c884e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4ffccbf60f4340da9d8be00a215c884e 2024-12-05T11:15:13,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4ffccbf60f4340da9d8be00a215c884e, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:15:13,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 04e661eeba0fa942008ec581fcd28e79 in 1270ms, sequenceid=60, compaction requested=true 2024-12-05T11:15:13,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:13,323 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:13,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:13,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:13,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:13,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:13,324 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:13,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:13,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:13,324 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:13,324 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:13,324 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,325 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/e576f13a197b48d0b4665b555d23f535, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8e666398a9314713b96fe31eb4ec3e3f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/11d6b5652a2a4d6caf07a66da2ee485b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=32.9 K 2024-12-05T11:15:13,325 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e576f13a197b48d0b4665b555d23f535, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733397310245 2024-12-05T11:15:13,325 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:13,325 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e666398a9314713b96fe31eb4ec3e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733397310274 2024-12-05T11:15:13,325 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:13,325 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,326 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/11be532676324a039cca2577674245f1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d202b45c691c499fbfeb364ce685cf5e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/a9cef8039dc945fb9fd961e14df99f6d] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=32.9 K 2024-12-05T11:15:13,326 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d6b5652a2a4d6caf07a66da2ee485b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397311421 2024-12-05T11:15:13,326 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 11be532676324a039cca2577674245f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733397310245 2024-12-05T11:15:13,326 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d202b45c691c499fbfeb364ce685cf5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733397310274 2024-12-05T11:15:13,327 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting a9cef8039dc945fb9fd961e14df99f6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397311421 2024-12-05T11:15:13,337 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:13,337 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/bacc119f5a1f42a480fed2eba2ee102c is 50, key is test_row_0/A:col10/1733397312053/Put/seqid=0 2024-12-05T11:15:13,338 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:13,339 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/995e8da5f2a14f43be0b1a0be80444dd is 50, key is test_row_0/B:col10/1733397312053/Put/seqid=0 2024-12-05T11:15:13,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742053_1229 (size=12104) 2024-12-05T11:15:13,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742054_1230 (size=12104) 2024-12-05T11:15:13,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T11:15:13,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,463 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:15:13,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4d5d5002795d48769ff5de7307568932 is 50, key is test_row_0/A:col10/1733397312072/Put/seqid=0 2024-12-05T11:15:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:15:13,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742055_1231 (size=12001) 2024-12-05T11:15:13,758 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/bacc119f5a1f42a480fed2eba2ee102c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/bacc119f5a1f42a480fed2eba2ee102c 2024-12-05T11:15:13,763 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into bacc119f5a1f42a480fed2eba2ee102c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:13,763 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:13,763 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397313323; duration=0sec 2024-12-05T11:15:13,763 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:13,763 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:13,763 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:13,765 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/995e8da5f2a14f43be0b1a0be80444dd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/995e8da5f2a14f43be0b1a0be80444dd 2024-12-05T11:15:13,765 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:13,765 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:13,765 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:13,765 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/7d06e70ea99e4835af72b467c15b3dbd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/244150eef7ce4d81a854dc278a6c6133, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4ffccbf60f4340da9d8be00a215c884e] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=32.9 K 2024-12-05T11:15:13,766 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d06e70ea99e4835af72b467c15b3dbd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733397310245 2024-12-05T11:15:13,766 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244150eef7ce4d81a854dc278a6c6133, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733397310274 2024-12-05T11:15:13,766 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffccbf60f4340da9d8be00a215c884e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397311421 2024-12-05T11:15:13,770 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into 995e8da5f2a14f43be0b1a0be80444dd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:13,770 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:13,770 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397313323; duration=0sec 2024-12-05T11:15:13,770 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:13,770 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:13,774 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:13,774 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/a36e1a62446e40efafa06b753db63bd3 is 50, key is test_row_0/C:col10/1733397312053/Put/seqid=0 2024-12-05T11:15:13,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742056_1232 (size=12104) 2024-12-05T11:15:13,792 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/a36e1a62446e40efafa06b753db63bd3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a36e1a62446e40efafa06b753db63bd3 2024-12-05T11:15:13,798 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into a36e1a62446e40efafa06b753db63bd3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:13,798 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:13,798 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=13, startTime=1733397313323; duration=0sec 2024-12-05T11:15:13,798 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:13,798 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:13,928 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4d5d5002795d48769ff5de7307568932 2024-12-05T11:15:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/85400207bbc5462783dd5d073223bd08 is 50, key is test_row_0/B:col10/1733397312072/Put/seqid=0 2024-12-05T11:15:13,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742057_1233 (size=12001) 2024-12-05T11:15:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397374236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397374237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397374237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397374239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397374239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,241 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,342 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/85400207bbc5462783dd5d073223bd08 2024-12-05T11:15:14,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397374349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397374349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397374349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397374349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/9703d158aa7948d7aae9aed2aeba6e73 is 50, key is test_row_0/C:col10/1733397312072/Put/seqid=0 2024-12-05T11:15:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742058_1234 (size=12001) 2024-12-05T11:15:14,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/9703d158aa7948d7aae9aed2aeba6e73 2024-12-05T11:15:14,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4d5d5002795d48769ff5de7307568932 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4d5d5002795d48769ff5de7307568932 2024-12-05T11:15:14,402 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4d5d5002795d48769ff5de7307568932, entries=150, sequenceid=80, filesize=11.7 K 2024-12-05T11:15:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/85400207bbc5462783dd5d073223bd08 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/85400207bbc5462783dd5d073223bd08 2024-12-05T11:15:14,407 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/85400207bbc5462783dd5d073223bd08, entries=150, sequenceid=80, filesize=11.7 K 2024-12-05T11:15:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/9703d158aa7948d7aae9aed2aeba6e73 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/9703d158aa7948d7aae9aed2aeba6e73 2024-12-05T11:15:14,414 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/9703d158aa7948d7aae9aed2aeba6e73, entries=150, sequenceid=80, filesize=11.7 K 2024-12-05T11:15:14,415 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 04e661eeba0fa942008ec581fcd28e79 in 952ms, sequenceid=80, compaction requested=false 2024-12-05T11:15:14,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:14,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-05T11:15:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-05T11:15:14,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T11:15:14,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0260 sec 2024-12-05T11:15:14,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.0330 sec 2024-12-05T11:15:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T11:15:14,539 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:14,539 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-05T11:15:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:14,541 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:14,541 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:14,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:14,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:15:14,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:14,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:14,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:14,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:14,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:14,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:14,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/0100bb40ad5a4b0cb66528e07279892d is 50, key is test_row_0/A:col10/1733397314236/Put/seqid=0 2024-12-05T11:15:14,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742059_1235 (size=14341) 2024-12-05T11:15:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397374575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397374577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397374579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397374579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:14,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397374688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397374688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397374688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397374688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:14,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:14,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397374898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397374898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397374898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397374898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:14,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/0100bb40ad5a4b0cb66528e07279892d 2024-12-05T11:15:14,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/482b4be21cc345d0bf92792783f39109 is 50, key is test_row_0/B:col10/1733397314236/Put/seqid=0 2024-12-05T11:15:14,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742060_1236 (size=12001) 2024-12-05T11:15:14,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:14,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:14,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:14,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:14,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:15,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:15,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:15,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397375208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397375208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397375208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397375208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/482b4be21cc345d0bf92792783f39109 2024-12-05T11:15:15,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/b8646d0109e049dca16083eb7fcc9a37 is 50, key is test_row_0/C:col10/1733397314236/Put/seqid=0 2024-12-05T11:15:15,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742061_1237 (size=12001) 2024-12-05T11:15:15,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397375718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397375718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397375718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397375718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:15,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:15,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/b8646d0109e049dca16083eb7fcc9a37 2024-12-05T11:15:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/0100bb40ad5a4b0cb66528e07279892d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/0100bb40ad5a4b0cb66528e07279892d 2024-12-05T11:15:15,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/0100bb40ad5a4b0cb66528e07279892d, entries=200, sequenceid=97, filesize=14.0 K 2024-12-05T11:15:15,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/482b4be21cc345d0bf92792783f39109 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/482b4be21cc345d0bf92792783f39109 2024-12-05T11:15:15,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/482b4be21cc345d0bf92792783f39109, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T11:15:15,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/b8646d0109e049dca16083eb7fcc9a37 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/b8646d0109e049dca16083eb7fcc9a37 2024-12-05T11:15:15,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/b8646d0109e049dca16083eb7fcc9a37, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T11:15:15,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 04e661eeba0fa942008ec581fcd28e79 in 1249ms, sequenceid=97, compaction requested=true 2024-12-05T11:15:15,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:15,810 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:15,810 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:15,810 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:15,810 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:15,811 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,811 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:15,811 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:15,811 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/bacc119f5a1f42a480fed2eba2ee102c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4d5d5002795d48769ff5de7307568932, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/0100bb40ad5a4b0cb66528e07279892d] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=37.5 K 2024-12-05T11:15:15,811 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,811 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/995e8da5f2a14f43be0b1a0be80444dd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/85400207bbc5462783dd5d073223bd08, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/482b4be21cc345d0bf92792783f39109] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=35.3 K 2024-12-05T11:15:15,811 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 995e8da5f2a14f43be0b1a0be80444dd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397311421 2024-12-05T11:15:15,811 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting bacc119f5a1f42a480fed2eba2ee102c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397311421 2024-12-05T11:15:15,812 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 85400207bbc5462783dd5d073223bd08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733397312061 2024-12-05T11:15:15,812 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d5d5002795d48769ff5de7307568932, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733397312061 2024-12-05T11:15:15,812 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 482b4be21cc345d0bf92792783f39109, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397314236 2024-12-05T11:15:15,812 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0100bb40ad5a4b0cb66528e07279892d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397314236 2024-12-05T11:15:15,818 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:15,818 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/b93f72aa1b934540b4f6f44945f0ab4f is 50, key is test_row_0/A:col10/1733397314236/Put/seqid=0 2024-12-05T11:15:15,818 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:15,819 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1e11b23fba1e457bb829daf89a40598e is 50, key is test_row_0/B:col10/1733397314236/Put/seqid=0 2024-12-05T11:15:15,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742063_1239 (size=12207) 2024-12-05T11:15:15,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742062_1238 (size=12207) 2024-12-05T11:15:15,830 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/b93f72aa1b934540b4f6f44945f0ab4f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/b93f72aa1b934540b4f6f44945f0ab4f 2024-12-05T11:15:15,835 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into b93f72aa1b934540b4f6f44945f0ab4f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:15,835 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:15,835 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397315809; duration=0sec 2024-12-05T11:15:15,835 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:15,835 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:15,836 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:15,837 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:15,837 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:15,838 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,838 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a36e1a62446e40efafa06b753db63bd3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/9703d158aa7948d7aae9aed2aeba6e73, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/b8646d0109e049dca16083eb7fcc9a37] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=35.3 K 2024-12-05T11:15:15,838 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36e1a62446e40efafa06b753db63bd3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397311421 2024-12-05T11:15:15,839 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9703d158aa7948d7aae9aed2aeba6e73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733397312061 2024-12-05T11:15:15,839 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8646d0109e049dca16083eb7fcc9a37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397314236 2024-12-05T11:15:15,847 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:15,848 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/2abb717aef5f4dbf9aaa737d5682ec90 is 50, key is test_row_0/C:col10/1733397314236/Put/seqid=0 2024-12-05T11:15:15,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742064_1240 (size=12207) 2024-12-05T11:15:15,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:15,916 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:15,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/6e19818b684a40e29b919dcc28cb8177 is 50, key is test_row_0/A:col10/1733397314576/Put/seqid=0 2024-12-05T11:15:15,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742065_1241 (size=12001) 2024-12-05T11:15:16,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1e11b23fba1e457bb829daf89a40598e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e11b23fba1e457bb829daf89a40598e 2024-12-05T11:15:16,231 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into 1e11b23fba1e457bb829daf89a40598e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:16,231 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:16,232 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397315810; duration=0sec 2024-12-05T11:15:16,232 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:16,232 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:16,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:16,266 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/2abb717aef5f4dbf9aaa737d5682ec90 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/2abb717aef5f4dbf9aaa737d5682ec90 2024-12-05T11:15:16,272 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into 2abb717aef5f4dbf9aaa737d5682ec90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:16,272 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:16,272 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=13, startTime=1733397315810; duration=0sec 2024-12-05T11:15:16,272 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:16,272 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397376296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,325 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/6e19818b684a40e29b919dcc28cb8177 2024-12-05T11:15:16,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/0ab5a352e1944612be06d2ffd096da68 is 50, key is test_row_0/B:col10/1733397314576/Put/seqid=0 2024-12-05T11:15:16,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742066_1242 (size=12001) 2024-12-05T11:15:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397376408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397376618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:16,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397376729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397376729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,731 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:16,731 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,739 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/0ab5a352e1944612be06d2ffd096da68 2024-12-05T11:15:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397376738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397376738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:16,741 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,741 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/d33bd49de5324f98b75a2cbbd931f7e5 is 50, key is test_row_0/C:col10/1733397314576/Put/seqid=0 2024-12-05T11:15:16,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742067_1243 (size=12001) 2024-12-05T11:15:16,775 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/d33bd49de5324f98b75a2cbbd931f7e5 2024-12-05T11:15:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/6e19818b684a40e29b919dcc28cb8177 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/6e19818b684a40e29b919dcc28cb8177 2024-12-05T11:15:16,790 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/6e19818b684a40e29b919dcc28cb8177, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T11:15:16,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/0ab5a352e1944612be06d2ffd096da68 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/0ab5a352e1944612be06d2ffd096da68 2024-12-05T11:15:16,796 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/0ab5a352e1944612be06d2ffd096da68, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T11:15:16,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/d33bd49de5324f98b75a2cbbd931f7e5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d33bd49de5324f98b75a2cbbd931f7e5 2024-12-05T11:15:16,803 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d33bd49de5324f98b75a2cbbd931f7e5, entries=150, sequenceid=116, filesize=11.7 K 2024-12-05T11:15:16,804 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 04e661eeba0fa942008ec581fcd28e79 in 888ms, sequenceid=116, compaction requested=false 2024-12-05T11:15:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-05T11:15:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-05T11:15:16,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T11:15:16,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2640 sec 2024-12-05T11:15:16,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.2700 sec 2024-12-05T11:15:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:16,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T11:15:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:16,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/550abafa1884492d844d63eb18285e7c is 50, key is test_row_0/A:col10/1733397316929/Put/seqid=0 2024-12-05T11:15:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742068_1244 (size=14541) 2024-12-05T11:15:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397377003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:17,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397377118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:17,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397377328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:17,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/550abafa1884492d844d63eb18285e7c 2024-12-05T11:15:17,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/544586ff90e8426d843554a2af01f497 is 50, key is test_row_0/B:col10/1733397316929/Put/seqid=0 2024-12-05T11:15:17,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742069_1245 (size=12151) 2024-12-05T11:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397377638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:17,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/544586ff90e8426d843554a2af01f497 2024-12-05T11:15:17,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/dea10971b1794ecaaa21c83ba5267c15 is 50, key is test_row_0/C:col10/1733397316929/Put/seqid=0 2024-12-05T11:15:17,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742070_1246 (size=12151) 2024-12-05T11:15:17,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/dea10971b1794ecaaa21c83ba5267c15 2024-12-05T11:15:17,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/550abafa1884492d844d63eb18285e7c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/550abafa1884492d844d63eb18285e7c 2024-12-05T11:15:17,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/550abafa1884492d844d63eb18285e7c, entries=200, sequenceid=133, filesize=14.2 K 2024-12-05T11:15:17,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/544586ff90e8426d843554a2af01f497 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/544586ff90e8426d843554a2af01f497 2024-12-05T11:15:17,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/544586ff90e8426d843554a2af01f497, entries=150, sequenceid=133, filesize=11.9 K 2024-12-05T11:15:17,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/dea10971b1794ecaaa21c83ba5267c15 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/dea10971b1794ecaaa21c83ba5267c15 2024-12-05T11:15:17,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/dea10971b1794ecaaa21c83ba5267c15, entries=150, sequenceid=133, filesize=11.9 K 2024-12-05T11:15:17,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 04e661eeba0fa942008ec581fcd28e79 in 874ms, sequenceid=133, compaction requested=true 2024-12-05T11:15:17,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:17,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:17,819 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:17,819 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:17,819 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:17,819 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:17,819 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:17,819 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:17,819 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:17,820 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:17,820 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e11b23fba1e457bb829daf89a40598e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/0ab5a352e1944612be06d2ffd096da68, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/544586ff90e8426d843554a2af01f497] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=35.5 K 2024-12-05T11:15:17,820 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/b93f72aa1b934540b4f6f44945f0ab4f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/6e19818b684a40e29b919dcc28cb8177, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/550abafa1884492d844d63eb18285e7c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=37.8 K 2024-12-05T11:15:17,820 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e11b23fba1e457bb829daf89a40598e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397314236 2024-12-05T11:15:17,820 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93f72aa1b934540b4f6f44945f0ab4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397314236 2024-12-05T11:15:17,820 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e19818b684a40e29b919dcc28cb8177, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733397314576 2024-12-05T11:15:17,820 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ab5a352e1944612be06d2ffd096da68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733397314576 2024-12-05T11:15:17,820 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550abafa1884492d844d63eb18285e7c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397316288 2024-12-05T11:15:17,821 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 544586ff90e8426d843554a2af01f497, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397316288 2024-12-05T11:15:17,826 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:17,826 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8d457c797f6a4d16a3b050c2ced9a4ce is 50, key is test_row_0/A:col10/1733397316929/Put/seqid=0 2024-12-05T11:15:17,831 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:17,831 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/d1aa0974478d48869e96d83fb8d9fe90 is 50, key is test_row_0/B:col10/1733397316929/Put/seqid=0 2024-12-05T11:15:17,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742071_1247 (size=12459) 2024-12-05T11:15:17,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742072_1248 (size=12459) 2024-12-05T11:15:17,846 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8d457c797f6a4d16a3b050c2ced9a4ce as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8d457c797f6a4d16a3b050c2ced9a4ce 2024-12-05T11:15:17,851 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into 8d457c797f6a4d16a3b050c2ced9a4ce(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:17,851 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:17,851 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397317818; duration=0sec 2024-12-05T11:15:17,851 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:17,851 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:17,851 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:17,852 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:17,852 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:17,852 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:17,853 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/2abb717aef5f4dbf9aaa737d5682ec90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d33bd49de5324f98b75a2cbbd931f7e5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/dea10971b1794ecaaa21c83ba5267c15] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=35.5 K 2024-12-05T11:15:17,853 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2abb717aef5f4dbf9aaa737d5682ec90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733397314236 2024-12-05T11:15:17,853 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d33bd49de5324f98b75a2cbbd931f7e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733397314576 2024-12-05T11:15:17,854 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea10971b1794ecaaa21c83ba5267c15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397316288 2024-12-05T11:15:17,867 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:17,868 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c382bf6556924652b4621c53ce7ca7b0 is 50, key is test_row_0/C:col10/1733397316929/Put/seqid=0 2024-12-05T11:15:17,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742073_1249 (size=12459) 2024-12-05T11:15:17,881 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c382bf6556924652b4621c53ce7ca7b0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c382bf6556924652b4621c53ce7ca7b0 2024-12-05T11:15:17,886 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into c382bf6556924652b4621c53ce7ca7b0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:17,886 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:17,886 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=13, startTime=1733397317819; duration=0sec 2024-12-05T11:15:17,887 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:17,887 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:18,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T11:15:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:18,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/181551588ce54aef8c3fc0a61e12440f is 50, key is test_row_0/A:col10/1733397318149/Put/seqid=0 2024-12-05T11:15:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742074_1250 (size=14541) 2024-12-05T11:15:18,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397378198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,240 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/d1aa0974478d48869e96d83fb8d9fe90 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d1aa0974478d48869e96d83fb8d9fe90 2024-12-05T11:15:18,252 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into d1aa0974478d48869e96d83fb8d9fe90(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:18,252 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:18,252 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397317819; duration=0sec 2024-12-05T11:15:18,252 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:18,252 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397378308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397378519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/181551588ce54aef8c3fc0a61e12440f 2024-12-05T11:15:18,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1e0a8b9788844ee6b70b3d6ef4a5a7de is 50, key is test_row_0/B:col10/1733397318149/Put/seqid=0 2024-12-05T11:15:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742075_1251 (size=12151) 2024-12-05T11:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T11:15:18,698 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:18,699 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-05T11:15:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:15:18,701 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:18,701 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:18,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:18,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397378738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,740 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397378749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,751 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397378758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397378758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,761 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,761 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:15:18,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397378828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:18,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:15:18,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:18,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:18,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:18,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:15:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:15:19,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1e0a8b9788844ee6b70b3d6ef4a5a7de 2024-12-05T11:15:19,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/d4a22baad1aa4698b047aefba38e8fca is 50, key is test_row_0/C:col10/1733397318149/Put/seqid=0 2024-12-05T11:15:19,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742076_1252 (size=12151) 2024-12-05T11:15:19,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:15:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:15:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:15:19,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397379338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:19,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/d4a22baad1aa4698b047aefba38e8fca 2024-12-05T11:15:19,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/181551588ce54aef8c3fc0a61e12440f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/181551588ce54aef8c3fc0a61e12440f 2024-12-05T11:15:19,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/181551588ce54aef8c3fc0a61e12440f, entries=200, sequenceid=153, filesize=14.2 K 2024-12-05T11:15:19,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1e0a8b9788844ee6b70b3d6ef4a5a7de as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e0a8b9788844ee6b70b3d6ef4a5a7de 2024-12-05T11:15:19,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e0a8b9788844ee6b70b3d6ef4a5a7de, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T11:15:19,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/d4a22baad1aa4698b047aefba38e8fca as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d4a22baad1aa4698b047aefba38e8fca 2024-12-05T11:15:19,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d4a22baad1aa4698b047aefba38e8fca, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T11:15:19,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 04e661eeba0fa942008ec581fcd28e79 in 1300ms, sequenceid=153, compaction requested=false 2024-12-05T11:15:19,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:19,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T11:15:19,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:19,465 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:15:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/85e97ce899534a999addfaf45f840f11 is 50, key is test_row_0/A:col10/1733397318196/Put/seqid=0 2024-12-05T11:15:19,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742077_1253 (size=12151) 2024-12-05T11:15:19,478 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/85e97ce899534a999addfaf45f840f11 2024-12-05T11:15:19,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/abb619744605459eb53fe5b7dd91ff69 is 50, key is test_row_0/B:col10/1733397318196/Put/seqid=0 2024-12-05T11:15:19,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742078_1254 (size=12151) 2024-12-05T11:15:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:15:19,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/abb619744605459eb53fe5b7dd91ff69 2024-12-05T11:15:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/61c6016d81b2405591a8cc3eacbab2c7 is 50, key is test_row_0/C:col10/1733397318196/Put/seqid=0 2024-12-05T11:15:19,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742079_1255 (size=12151) 2024-12-05T11:15:20,305 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/61c6016d81b2405591a8cc3eacbab2c7 2024-12-05T11:15:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/85e97ce899534a999addfaf45f840f11 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/85e97ce899534a999addfaf45f840f11 2024-12-05T11:15:20,312 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/85e97ce899534a999addfaf45f840f11, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T11:15:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/abb619744605459eb53fe5b7dd91ff69 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/abb619744605459eb53fe5b7dd91ff69 2024-12-05T11:15:20,316 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/abb619744605459eb53fe5b7dd91ff69, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T11:15:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/61c6016d81b2405591a8cc3eacbab2c7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/61c6016d81b2405591a8cc3eacbab2c7 2024-12-05T11:15:20,321 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/61c6016d81b2405591a8cc3eacbab2c7, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T11:15:20,322 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 04e661eeba0fa942008ec581fcd28e79 in 857ms, sequenceid=168, compaction requested=true 2024-12-05T11:15:20,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:20,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:20,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-05T11:15:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-05T11:15:20,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-05T11:15:20,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-12-05T11:15:20,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6260 sec 2024-12-05T11:15:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:20,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:15:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:20,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:20,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8a8c4870914148b1b089ec62c735aae3 is 50, key is test_row_0/A:col10/1733397320356/Put/seqid=0 2024-12-05T11:15:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742080_1256 (size=12151) 2024-12-05T11:15:20,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8a8c4870914148b1b089ec62c735aae3 2024-12-05T11:15:20,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/f7997c78d0a04edaa029dddb96c99736 is 50, key is test_row_0/B:col10/1733397320356/Put/seqid=0 2024-12-05T11:15:20,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742081_1257 (size=12151) 2024-12-05T11:15:20,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/f7997c78d0a04edaa029dddb96c99736 2024-12-05T11:15:20,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/4c67dc1dd9a641f88541cd3a967ed114 is 50, key is test_row_0/C:col10/1733397320356/Put/seqid=0 2024-12-05T11:15:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742082_1258 (size=12151) 2024-12-05T11:15:20,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397320411, max timestamp=1733397320426Num uniques 859; 2024-12-05T11:15:20,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397320411, max timestamp=1733397320426Num uniques 859; 2024-12-05T11:15:20,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:20,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:20,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397320411, max timestamp=1733397320426Num uniques 859; 2024-12-05T11:15:20,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397380498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397380608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:20,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/4c67dc1dd9a641f88541cd3a967ed114 2024-12-05T11:15:20,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/8a8c4870914148b1b089ec62c735aae3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8a8c4870914148b1b089ec62c735aae3 2024-12-05T11:15:20,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8a8c4870914148b1b089ec62c735aae3, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T11:15:20,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/f7997c78d0a04edaa029dddb96c99736 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f7997c78d0a04edaa029dddb96c99736 2024-12-05T11:15:20,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f7997c78d0a04edaa029dddb96c99736, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T11:15:20,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/4c67dc1dd9a641f88541cd3a967ed114 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4c67dc1dd9a641f88541cd3a967ed114 2024-12-05T11:15:20,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4c67dc1dd9a641f88541cd3a967ed114, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T11:15:20,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 04e661eeba0fa942008ec581fcd28e79 in 455ms, sequenceid=178, compaction requested=true 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:20,817 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:20,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:15:20,818 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:20,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397320448, max timestamp=1733397320818Num uniques 1018; 2024-12-05T11:15:20,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397320448, max timestamp=1733397320818Num uniques 1018; 2024-12-05T11:15:20,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:15:20,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:15:20,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-05T11:15:20,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397320448, max timestamp=1733397320818Num uniques 1018; 2024-12-05T11:15:20,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:15:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:20,823 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:20,823 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:20,823 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:20,823 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d1aa0974478d48869e96d83fb8d9fe90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e0a8b9788844ee6b70b3d6ef4a5a7de, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/abb619744605459eb53fe5b7dd91ff69, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f7997c78d0a04edaa029dddb96c99736] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=47.8 K 2024-12-05T11:15:20,823 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:20,824 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:20,824 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:20,824 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8d457c797f6a4d16a3b050c2ced9a4ce, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/181551588ce54aef8c3fc0a61e12440f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/85e97ce899534a999addfaf45f840f11, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8a8c4870914148b1b089ec62c735aae3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=50.1 K 2024-12-05T11:15:20,824 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d1aa0974478d48869e96d83fb8d9fe90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397316288 2024-12-05T11:15:20,824 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d457c797f6a4d16a3b050c2ced9a4ce, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397316288 2024-12-05T11:15:20,824 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0a8b9788844ee6b70b3d6ef4a5a7de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733397316987 2024-12-05T11:15:20,824 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181551588ce54aef8c3fc0a61e12440f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733397316987 2024-12-05T11:15:20,825 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting abb619744605459eb53fe5b7dd91ff69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733397318174 2024-12-05T11:15:20,825 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e97ce899534a999addfaf45f840f11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733397318174 2024-12-05T11:15:20,825 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f7997c78d0a04edaa029dddb96c99736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733397320356 2024-12-05T11:15:20,826 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8c4870914148b1b089ec62c735aae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733397320356 2024-12-05T11:15:20,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/57278e2e7a6d43908dc64ade87620a05 is 50, key is test_row_0/A:col10/1733397320818/Put/seqid=0 2024-12-05T11:15:20,839 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:20,839 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:20,839 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/66a3b24ac51b4e08860220168d96c4d1 is 50, key is test_row_0/A:col10/1733397320356/Put/seqid=0 2024-12-05T11:15:20,840 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/88aadded26b44551b04a3fdeab424d0c is 50, key is test_row_0/B:col10/1733397320356/Put/seqid=0 2024-12-05T11:15:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T11:15:20,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:20,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-05T11:15:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:20,852 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:20,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:20,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742083_1259 (size=12151) 2024-12-05T11:15:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742084_1260 (size=12595) 2024-12-05T11:15:20,875 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/66a3b24ac51b4e08860220168d96c4d1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/66a3b24ac51b4e08860220168d96c4d1 2024-12-05T11:15:20,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742085_1261 (size=12595) 2024-12-05T11:15:20,881 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into 66a3b24ac51b4e08860220168d96c4d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:20,881 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:20,881 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=12, startTime=1733397320817; duration=0sec 2024-12-05T11:15:20,881 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:20,881 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:20,882 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:20,883 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:20,884 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:20,884 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:20,884 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c382bf6556924652b4621c53ce7ca7b0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d4a22baad1aa4698b047aefba38e8fca, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/61c6016d81b2405591a8cc3eacbab2c7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4c67dc1dd9a641f88541cd3a967ed114] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=47.8 K 2024-12-05T11:15:20,884 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c382bf6556924652b4621c53ce7ca7b0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397316288 2024-12-05T11:15:20,885 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a22baad1aa4698b047aefba38e8fca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733397316987 2024-12-05T11:15:20,886 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c6016d81b2405591a8cc3eacbab2c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733397318174 2024-12-05T11:15:20,886 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c67dc1dd9a641f88541cd3a967ed114, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733397320356 2024-12-05T11:15:20,888 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/88aadded26b44551b04a3fdeab424d0c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/88aadded26b44551b04a3fdeab424d0c 2024-12-05T11:15:20,896 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into 88aadded26b44551b04a3fdeab424d0c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:20,896 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:20,896 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=12, startTime=1733397320817; duration=0sec 2024-12-05T11:15:20,896 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:20,896 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:20,903 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:20,904 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/74ecb637795b4c528287d68fa35979bb is 50, key is test_row_0/C:col10/1733397320356/Put/seqid=0 2024-12-05T11:15:20,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397320857, max timestamp=1733397320902Num uniques 859; 2024-12-05T11:15:20,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:20,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397320857, max timestamp=1733397320902Num uniques 859; 2024-12-05T11:15:20,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:20,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397320857, max timestamp=1733397320902Num uniques 859; 2024-12-05T11:15:20,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T11:15:20,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742086_1262 (size=12595) 2024-12-05T11:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:20,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733397320931, max timestamp=1733397320985Num uniques 1009; 2024-12-05T11:15:20,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733397320931, max timestamp=1733397320985Num uniques 1018; 2024-12-05T11:15:20,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733397320931, max timestamp=1733397320985Num uniques 1018; 2024-12-05T11:15:20,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T11:15:20,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-05T11:15:20,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T11:15:21,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733397321042, max timestamp=1733397321058Num uniques 1077; 2024-12-05T11:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733397321042, max timestamp=1733397321058Num uniques 1077; 2024-12-05T11:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-05T11:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-05T11:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733397321042, max timestamp=1733397321058Num uniques 1077; 2024-12-05T11:15:21,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-05T11:15:21,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733397321104, max timestamp=1733397321128Num uniques 1036; 2024-12-05T11:15:21,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-05T11:15:21,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733397321104, max timestamp=1733397321128Num uniques 1036; 2024-12-05T11:15:21,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-05T11:15:21,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733397321104, max timestamp=1733397321128Num uniques 1036; 2024-12-05T11:15:21,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-05T11:15:21,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:21,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733397321182, max timestamp=1733397321198Num uniques 1045; 2024-12-05T11:15:21,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-05T11:15:21,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733397321182, max timestamp=1733397321198Num uniques 1045; 2024-12-05T11:15:21,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-05T11:15:21,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733397321182, max timestamp=1733397321198Num uniques 1045; 2024-12-05T11:15:21,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-05T11:15:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733397321255, max timestamp=1733397321266Num uniques 1054; 2024-12-05T11:15:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-05T11:15:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733397321255, max timestamp=1733397321266Num uniques 1054; 2024-12-05T11:15:21,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/57278e2e7a6d43908dc64ade87620a05 2024-12-05T11:15:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-05T11:15:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733397321255, max timestamp=1733397321266Num uniques 1054; 2024-12-05T11:15:21,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-05T11:15:21,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/b088cf3ca7494416a40a684beeb9b1d2 is 50, key is test_row_0/B:col10/1733397320818/Put/seqid=0 2024-12-05T11:15:21,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742087_1263 (size=12151) 2024-12-05T11:15:21,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,315 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/74ecb637795b4c528287d68fa35979bb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/74ecb637795b4c528287d68fa35979bb 2024-12-05T11:15:21,321 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into 74ecb637795b4c528287d68fa35979bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:21,321 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:21,321 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=12, startTime=1733397320817; duration=0sec 2024-12-05T11:15:21,321 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:21,321 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:21,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733397321298, max timestamp=1733397321344Num uniques 1013; 2024-12-05T11:15:21,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-05T11:15:21,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733397321298, max timestamp=1733397321344Num uniques 1013; 2024-12-05T11:15:21,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-05T11:15:21,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733397321298, max timestamp=1733397321344Num uniques 1013; 2024-12-05T11:15:21,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-05T11:15:21,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733397321386, max timestamp=1733397321404Num uniques 1022; 2024-12-05T11:15:21,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-05T11:15:21,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733397321386, max timestamp=1733397321404Num uniques 1022; 2024-12-05T11:15:21,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733397321386, max timestamp=1733397321404Num uniques 1022; 2024-12-05T11:15:21,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-05T11:15:21,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-05T11:15:21,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733397321412, max timestamp=1733397321456Num uniques 1031; 2024-12-05T11:15:21,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-05T11:15:21,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733397321412, max timestamp=1733397321456Num uniques 1031; 2024-12-05T11:15:21,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-05T11:15:21,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733397321412, max timestamp=1733397321456Num uniques 1031; 2024-12-05T11:15:21,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-05T11:15:21,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:21,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733397321483, max timestamp=1733397321507Num uniques 1040; 2024-12-05T11:15:21,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-05T11:15:21,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733397321483, max timestamp=1733397321507Num uniques 1040; 2024-12-05T11:15:21,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-05T11:15:21,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733397321483, max timestamp=1733397321507Num uniques 1040; 2024-12-05T11:15:21,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-05T11:15:21,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733397321546, max timestamp=1733397321560Num uniques 1049; 2024-12-05T11:15:21,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733397321546, max timestamp=1733397321560Num uniques 1049; 2024-12-05T11:15:21,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-05T11:15:21,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-05T11:15:21,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733397321546, max timestamp=1733397321560Num uniques 1049; 2024-12-05T11:15:21,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-05T11:15:21,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733397321604, max timestamp=1733397321622Num uniques 1058; 2024-12-05T11:15:21,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733397321604, max timestamp=1733397321622Num uniques 1058; 2024-12-05T11:15:21,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-05T11:15:21,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-05T11:15:21,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733397321604, max timestamp=1733397321622Num uniques 1058; 2024-12-05T11:15:21,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-05T11:15:21,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733397321663, max timestamp=1733397321676Num uniques 1017; 2024-12-05T11:15:21,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-05T11:15:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733397321663, max timestamp=1733397321676Num uniques 1017; 2024-12-05T11:15:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-05T11:15:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733397321663, max timestamp=1733397321676Num uniques 1017; 2024-12-05T11:15:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-05T11:15:21,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/b088cf3ca7494416a40a684beeb9b1d2 2024-12-05T11:15:21,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c3534e19c3724cbd954e13f1112f9f91 is 50, key is test_row_0/C:col10/1733397320818/Put/seqid=0 2024-12-05T11:15:21,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742088_1264 (size=12151) 2024-12-05T11:15:21,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733397321714, max timestamp=1733397321731Num uniques 1026; 2024-12-05T11:15:21,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-05T11:15:21,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733397321714, max timestamp=1733397321731Num uniques 1026; 2024-12-05T11:15:21,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-05T11:15:21,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733397321714, max timestamp=1733397321731Num uniques 1026; 2024-12-05T11:15:21,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-05T11:15:21,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733397321778, max timestamp=1733397321792Num uniques 1035; 2024-12-05T11:15:21,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-05T11:15:21,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733397321778, max timestamp=1733397321792Num uniques 1035; 2024-12-05T11:15:21,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-05T11:15:21,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733397321778, max timestamp=1733397321792Num uniques 1035; 2024-12-05T11:15:21,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-05T11:15:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733397321843, max timestamp=1733397321853Num uniques 1044; 2024-12-05T11:15:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-05T11:15:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397321843, max timestamp=1733397321853Num uniques 1044; 2024-12-05T11:15:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=6870, suffix heap size=136544,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=150 2024-12-05T11:15:21,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733397321843, max timestamp=1733397321853Num uniques 1044; 2024-12-05T11:15:21,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-05T11:15:21,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733397321879, max timestamp=1733397321918Num uniques 1053; 2024-12-05T11:15:21,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-05T11:15:21,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733397321879, max timestamp=1733397321918Num uniques 1053; 2024-12-05T11:15:21,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-05T11:15:21,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733397321879, max timestamp=1733397321918Num uniques 1009; 2024-12-05T11:15:21,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=7005, suffix heap size=132904,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=153 2024-12-05T11:15:21,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:21,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:21,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:21,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733397321953, max timestamp=1733397321981Num uniques 1012; 2024-12-05T11:15:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-05T11:15:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733397321953, max timestamp=1733397321981Num uniques 1012; 2024-12-05T11:15:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733397321953, max timestamp=1733397321981Num uniques 1012; 2024-12-05T11:15:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-05T11:15:21,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=7412, suffix heap size=133216,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=162 2024-12-05T11:15:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:22,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733397322016, max timestamp=1733397322037Num uniques 1021; 2024-12-05T11:15:22,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733397322016, max timestamp=1733397322037Num uniques 1021; 2024-12-05T11:15:22,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733397322016, max timestamp=1733397322037Num uniques 1021; 2024-12-05T11:15:22,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-05T11:15:22,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-05T11:15:22,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=7826, suffix heap size=134152,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=171 2024-12-05T11:15:22,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733397322073, max timestamp=1733397322083Num uniques 1030; 2024-12-05T11:15:22,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-05T11:15:22,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733397322073, max timestamp=1733397322083Num uniques 1030; 2024-12-05T11:15:22,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-05T11:15:22,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1733397322073, max timestamp=1733397322083Num uniques 1030; 2024-12-05T11:15:22,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=8240, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-12-05T11:15:22,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c3534e19c3724cbd954e13f1112f9f91 2024-12-05T11:15:22,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/57278e2e7a6d43908dc64ade87620a05 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/57278e2e7a6d43908dc64ade87620a05 2024-12-05T11:15:22,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/57278e2e7a6d43908dc64ade87620a05, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T11:15:22,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/b088cf3ca7494416a40a684beeb9b1d2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/b088cf3ca7494416a40a684beeb9b1d2 2024-12-05T11:15:22,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/b088cf3ca7494416a40a684beeb9b1d2, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T11:15:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c3534e19c3724cbd954e13f1112f9f91 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c3534e19c3724cbd954e13f1112f9f91 2024-12-05T11:15:22,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c3534e19c3724cbd954e13f1112f9f91, entries=150, sequenceid=216, filesize=11.9 K 2024-12-05T11:15:22,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=60.38 KB/61830 for 04e661eeba0fa942008ec581fcd28e79 in 1294ms, sequenceid=216, compaction requested=false 2024-12-05T11:15:22,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:22,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=67.09 KB heapSize=183.80 KB 2024-12-05T11:15:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:22,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/eb8bda54ab0b4e9f8b93d0743b719848 is 50, key is test_row_0/A:col10/1733397322073/Put/seqid=0 2024-12-05T11:15:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742089_1265 (size=12301) 2024-12-05T11:15:22,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/eb8bda54ab0b4e9f8b93d0743b719848 2024-12-05T11:15:22,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/871f9f2e59ba483083d07b70a11ef393 is 50, key is test_row_0/B:col10/1733397322073/Put/seqid=0 2024-12-05T11:15:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742090_1266 (size=12301) 2024-12-05T11:15:22,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 488 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397382200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 489 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397382309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:22,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 490 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397382518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/871f9f2e59ba483083d07b70a11ef393 2024-12-05T11:15:22,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/86ed3e56f5de423aa8f641847aa5b2fb is 50, key is test_row_0/C:col10/1733397322073/Put/seqid=0 2024-12-05T11:15:22,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742091_1267 (size=12301) 2024-12-05T11:15:22,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58312 deadline: 1733397382758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,762 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58332 deadline: 1733397382768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,772 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58338 deadline: 1733397382779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,781 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58290 deadline: 1733397382788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,792 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in 04e661eeba0fa942008ec581fcd28e79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 491 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397382828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:22,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:22,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:22,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/86ed3e56f5de423aa8f641847aa5b2fb 2024-12-05T11:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/eb8bda54ab0b4e9f8b93d0743b719848 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/eb8bda54ab0b4e9f8b93d0743b719848 2024-12-05T11:15:22,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/eb8bda54ab0b4e9f8b93d0743b719848, entries=150, sequenceid=572, filesize=12.0 K 2024-12-05T11:15:22,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/871f9f2e59ba483083d07b70a11ef393 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/871f9f2e59ba483083d07b70a11ef393 2024-12-05T11:15:22,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/871f9f2e59ba483083d07b70a11ef393, entries=150, sequenceid=572, filesize=12.0 K 2024-12-05T11:15:22,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/86ed3e56f5de423aa8f641847aa5b2fb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86ed3e56f5de423aa8f641847aa5b2fb 2024-12-05T11:15:22,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86ed3e56f5de423aa8f641847aa5b2fb, entries=150, sequenceid=572, filesize=12.0 K 2024-12-05T11:15:22,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.75 KB/188160, currentSize=114.05 KB/116790 for 04e661eeba0fa942008ec581fcd28e79 in 874ms, sequenceid=572, compaction requested=true 2024-12-05T11:15:22,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:22,994 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:22,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T11:15:22,994 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:22,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:22,995 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:15:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:22,996 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:22,997 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:22,997 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,997 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/88aadded26b44551b04a3fdeab424d0c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/b088cf3ca7494416a40a684beeb9b1d2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/871f9f2e59ba483083d07b70a11ef393] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=36.2 K 2024-12-05T11:15:22,997 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:22,997 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:22,997 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:22,997 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 88aadded26b44551b04a3fdeab424d0c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733397320356 2024-12-05T11:15:22,998 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/66a3b24ac51b4e08860220168d96c4d1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/57278e2e7a6d43908dc64ade87620a05, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/eb8bda54ab0b4e9f8b93d0743b719848] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=36.2 K 2024-12-05T11:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:22,999 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b088cf3ca7494416a40a684beeb9b1d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733397320491 2024-12-05T11:15:22,999 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a3b24ac51b4e08860220168d96c4d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733397320356 2024-12-05T11:15:22,999 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 871f9f2e59ba483083d07b70a11ef393, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733397322073 2024-12-05T11:15:23,000 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57278e2e7a6d43908dc64ade87620a05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733397320491 2024-12-05T11:15:23,001 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb8bda54ab0b4e9f8b93d0743b719848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733397322073 2024-12-05T11:15:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4a7faf63c75e4d0c9e9092f92dc43f3e is 50, key is test_row_0/A:col10/1733397322199/Put/seqid=0 2024-12-05T11:15:23,022 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#226 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:23,022 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/56157d7f9f94493d8107094593b184e4 is 50, key is test_row_0/A:col10/1733397322073/Put/seqid=0 2024-12-05T11:15:23,038 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:23,038 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/7627b99febbe40418d9aff6045caca8c is 50, key is test_row_0/B:col10/1733397322073/Put/seqid=0 2024-12-05T11:15:23,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742092_1268 (size=12301) 2024-12-05T11:15:23,079 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4a7faf63c75e4d0c9e9092f92dc43f3e 2024-12-05T11:15:23,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742093_1269 (size=12847) 2024-12-05T11:15:23,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1fddf734351f4306ba30bb2e17033434 is 50, key is test_row_0/B:col10/1733397322199/Put/seqid=0 2024-12-05T11:15:23,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742094_1270 (size=12847) 2024-12-05T11:15:23,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742095_1271 (size=12301) 2024-12-05T11:15:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:23,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:23,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 501 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397383370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 502 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397383478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:23,503 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/56157d7f9f94493d8107094593b184e4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/56157d7f9f94493d8107094593b184e4 2024-12-05T11:15:23,507 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/7627b99febbe40418d9aff6045caca8c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/7627b99febbe40418d9aff6045caca8c 2024-12-05T11:15:23,507 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into 56157d7f9f94493d8107094593b184e4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:23,507 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:23,507 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397322993; duration=0sec 2024-12-05T11:15:23,507 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:23,507 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:23,508 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:23,509 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:23,509 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:23,509 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:23,509 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/74ecb637795b4c528287d68fa35979bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c3534e19c3724cbd954e13f1112f9f91, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86ed3e56f5de423aa8f641847aa5b2fb] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=36.2 K 2024-12-05T11:15:23,509 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ecb637795b4c528287d68fa35979bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733397320356 2024-12-05T11:15:23,510 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3534e19c3724cbd954e13f1112f9f91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733397320491 2024-12-05T11:15:23,510 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ed3e56f5de423aa8f641847aa5b2fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733397322073 2024-12-05T11:15:23,511 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into 7627b99febbe40418d9aff6045caca8c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:23,512 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:23,512 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397322994; duration=0sec 2024-12-05T11:15:23,512 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:23,513 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:23,516 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1fddf734351f4306ba30bb2e17033434 2024-12-05T11:15:23,523 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:23,524 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/3012a7aa2c744ed4beca5525b4dafa6b is 50, key is test_row_0/C:col10/1733397322073/Put/seqid=0 2024-12-05T11:15:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/8daada6be2ce451a9900059d224fccf7 is 50, key is test_row_0/C:col10/1733397322199/Put/seqid=0 2024-12-05T11:15:23,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742096_1272 (size=12847) 2024-12-05T11:15:23,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742097_1273 (size=12301) 2024-12-05T11:15:23,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 503 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397383689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:23,946 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/8daada6be2ce451a9900059d224fccf7 2024-12-05T11:15:23,951 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/3012a7aa2c744ed4beca5525b4dafa6b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/3012a7aa2c744ed4beca5525b4dafa6b 2024-12-05T11:15:23,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4a7faf63c75e4d0c9e9092f92dc43f3e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4a7faf63c75e4d0c9e9092f92dc43f3e 2024-12-05T11:15:23,963 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into 3012a7aa2c744ed4beca5525b4dafa6b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:23,963 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:23,963 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=13, startTime=1733397322994; duration=0sec 2024-12-05T11:15:23,964 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:23,964 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:23,965 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4a7faf63c75e4d0c9e9092f92dc43f3e, entries=150, sequenceid=592, filesize=12.0 K 2024-12-05T11:15:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1fddf734351f4306ba30bb2e17033434 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fddf734351f4306ba30bb2e17033434 2024-12-05T11:15:23,975 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fddf734351f4306ba30bb2e17033434, entries=150, sequenceid=592, filesize=12.0 K 2024-12-05T11:15:23,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/8daada6be2ce451a9900059d224fccf7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/8daada6be2ce451a9900059d224fccf7 2024-12-05T11:15:23,993 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/8daada6be2ce451a9900059d224fccf7, entries=150, sequenceid=592, filesize=12.0 K 2024-12-05T11:15:23,994 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 04e661eeba0fa942008ec581fcd28e79 in 1000ms, sequenceid=592, compaction requested=false 2024-12-05T11:15:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-05T11:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-05T11:15:24,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-05T11:15:24,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1430 sec 2024-12-05T11:15:24,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:15:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:24,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.1560 sec 2024-12-05T11:15:24,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/3755a53ec71a441c91a29af16619fd03 is 50, key is test_row_1/A:col10/1733397323999/Put/seqid=0 2024-12-05T11:15:24,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742098_1274 (size=9857) 2024-12-05T11:15:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 521 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397384053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 522 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397384158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 523 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397384368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:24,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/3755a53ec71a441c91a29af16619fd03 2024-12-05T11:15:24,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1fc96ce124024d6d97952b65c98bcde1 is 50, key is test_row_1/B:col10/1733397323999/Put/seqid=0 2024-12-05T11:15:24,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742099_1275 (size=9857) 2024-12-05T11:15:24,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 524 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397384678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:24,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1fc96ce124024d6d97952b65c98bcde1 2024-12-05T11:15:24,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/37424dd030d44598846e56764f8b0ac7 is 50, key is test_row_1/C:col10/1733397323999/Put/seqid=0 2024-12-05T11:15:24,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742100_1276 (size=9857) 2024-12-05T11:15:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T11:15:25,018 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:25,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-05T11:15:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:15:25,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:25,021 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:25,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:15:25,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:15:25,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:25,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 525 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397385188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:25,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/37424dd030d44598846e56764f8b0ac7 2024-12-05T11:15:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/3755a53ec71a441c91a29af16619fd03 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3755a53ec71a441c91a29af16619fd03 2024-12-05T11:15:25,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3755a53ec71a441c91a29af16619fd03, entries=100, sequenceid=608, filesize=9.6 K 2024-12-05T11:15:25,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/1fc96ce124024d6d97952b65c98bcde1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fc96ce124024d6d97952b65c98bcde1 2024-12-05T11:15:25,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fc96ce124024d6d97952b65c98bcde1, entries=100, sequenceid=608, filesize=9.6 K 2024-12-05T11:15:25,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/37424dd030d44598846e56764f8b0ac7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/37424dd030d44598846e56764f8b0ac7 2024-12-05T11:15:25,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/37424dd030d44598846e56764f8b0ac7, entries=100, sequenceid=608, filesize=9.6 K 2024-12-05T11:15:25,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 04e661eeba0fa942008ec581fcd28e79 in 1276ms, sequenceid=608, compaction requested=true 2024-12-05T11:15:25,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:25,276 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:25,277 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:25,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:25,278 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:25,278 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:25,278 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:25,278 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:25,278 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,278 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,278 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/7627b99febbe40418d9aff6045caca8c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fddf734351f4306ba30bb2e17033434, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fc96ce124024d6d97952b65c98bcde1] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=34.2 K 2024-12-05T11:15:25,278 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/56157d7f9f94493d8107094593b184e4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4a7faf63c75e4d0c9e9092f92dc43f3e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3755a53ec71a441c91a29af16619fd03] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=34.2 K 2024-12-05T11:15:25,279 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7627b99febbe40418d9aff6045caca8c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733397322073 2024-12-05T11:15:25,279 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56157d7f9f94493d8107094593b184e4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733397322073 2024-12-05T11:15:25,279 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fddf734351f4306ba30bb2e17033434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1733397322178 2024-12-05T11:15:25,279 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a7faf63c75e4d0c9e9092f92dc43f3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1733397322178 2024-12-05T11:15:25,280 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fc96ce124024d6d97952b65c98bcde1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733397323363 2024-12-05T11:15:25,280 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3755a53ec71a441c91a29af16619fd03, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733397323363 2024-12-05T11:15:25,298 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:25,299 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/ebe97e243d77401f87621ab5107bc850 is 50, key is test_row_0/A:col10/1733397322199/Put/seqid=0 2024-12-05T11:15:25,300 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:25,300 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/21e056463f8e447c8812da4cf2deebd2 is 50, key is test_row_0/B:col10/1733397322199/Put/seqid=0 2024-12-05T11:15:25,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T11:15:25,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,328 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:15:25,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:25,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:25,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742102_1278 (size=12949) 2024-12-05T11:15:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:15:25,345 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/21e056463f8e447c8812da4cf2deebd2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/21e056463f8e447c8812da4cf2deebd2 2024-12-05T11:15:25,352 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into 21e056463f8e447c8812da4cf2deebd2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:25,352 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:25,352 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397325277; duration=0sec 2024-12-05T11:15:25,352 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:25,352 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:25,352 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:25,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742101_1277 (size=12949) 2024-12-05T11:15:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/075ecba491244e149d12d9dbc9c2e8d3 is 50, key is test_row_0/A:col10/1733397324051/Put/seqid=0 2024-12-05T11:15:25,359 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:25,359 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:25,359 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,359 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/3012a7aa2c744ed4beca5525b4dafa6b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/8daada6be2ce451a9900059d224fccf7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/37424dd030d44598846e56764f8b0ac7] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=34.2 K 2024-12-05T11:15:25,360 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3012a7aa2c744ed4beca5525b4dafa6b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1733397322073 2024-12-05T11:15:25,361 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 8daada6be2ce451a9900059d224fccf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1733397322178 2024-12-05T11:15:25,363 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 37424dd030d44598846e56764f8b0ac7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733397323363 2024-12-05T11:15:25,364 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/ebe97e243d77401f87621ab5107bc850 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/ebe97e243d77401f87621ab5107bc850 2024-12-05T11:15:25,372 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into ebe97e243d77401f87621ab5107bc850(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:25,372 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:25,372 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397325276; duration=0sec 2024-12-05T11:15:25,372 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:25,372 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:25,382 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:25,383 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/00201c8931d54e8989156bc7776cde60 is 50, key is test_row_0/C:col10/1733397322199/Put/seqid=0 2024-12-05T11:15:25,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742103_1279 (size=12301) 2024-12-05T11:15:25,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742104_1280 (size=12949) 2024-12-05T11:15:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:15:25,786 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/075ecba491244e149d12d9dbc9c2e8d3 2024-12-05T11:15:25,813 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/00201c8931d54e8989156bc7776cde60 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/00201c8931d54e8989156bc7776cde60 2024-12-05T11:15:25,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/f0fef308140c404d97c96991a472c82b is 50, key is test_row_0/B:col10/1733397324051/Put/seqid=0 2024-12-05T11:15:25,820 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into 00201c8931d54e8989156bc7776cde60(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:25,820 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:25,820 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=13, startTime=1733397325277; duration=0sec 2024-12-05T11:15:25,820 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:25,820 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742105_1281 (size=12301) 2024-12-05T11:15:25,856 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/f0fef308140c404d97c96991a472c82b 2024-12-05T11:15:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/1f40380bbdf74dea88da85a704578856 is 50, key is test_row_0/C:col10/1733397324051/Put/seqid=0 2024-12-05T11:15:25,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742106_1282 (size=12301) 2024-12-05T11:15:25,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/1f40380bbdf74dea88da85a704578856 2024-12-05T11:15:25,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/075ecba491244e149d12d9dbc9c2e8d3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/075ecba491244e149d12d9dbc9c2e8d3 2024-12-05T11:15:25,945 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/075ecba491244e149d12d9dbc9c2e8d3, entries=150, sequenceid=627, filesize=12.0 K 2024-12-05T11:15:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/f0fef308140c404d97c96991a472c82b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f0fef308140c404d97c96991a472c82b 2024-12-05T11:15:25,955 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f0fef308140c404d97c96991a472c82b, entries=150, sequenceid=627, filesize=12.0 K 2024-12-05T11:15:25,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/1f40380bbdf74dea88da85a704578856 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/1f40380bbdf74dea88da85a704578856 2024-12-05T11:15:25,963 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/1f40380bbdf74dea88da85a704578856, entries=150, sequenceid=627, filesize=12.0 K 2024-12-05T11:15:25,964 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 04e661eeba0fa942008ec581fcd28e79 in 636ms, sequenceid=627, compaction requested=false 2024-12-05T11:15:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-05T11:15:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-05T11:15:25,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-05T11:15:25,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 945 msec 2024-12-05T11:15:25,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 950 msec 2024-12-05T11:15:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T11:15:26,163 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:26,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-05T11:15:26,166 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:15:26,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:26,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:26,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:15:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:26,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/5154e89591934eae963e6d0f6b31c172 is 50, key is test_row_0/A:col10/1733397326224/Put/seqid=0 2024-12-05T11:15:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:15:26,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742107_1283 (size=14741) 2024-12-05T11:15:26,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/5154e89591934eae963e6d0f6b31c172 2024-12-05T11:15:26,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/59cae3d732d6462cadda6bc4523fc1bb is 50, key is test_row_0/B:col10/1733397326224/Put/seqid=0 2024-12-05T11:15:26,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397326263, max timestamp=1733397326319Num uniques 859; 2024-12-05T11:15:26,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397326263, max timestamp=1733397326319Num uniques 859; 2024-12-05T11:15:26,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:26,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:26,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397326263, max timestamp=1733397326319Num uniques 859; 2024-12-05T11:15:26,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:26,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742108_1284 (size=12301) 2024-12-05T11:15:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 567 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397386414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:26,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:15:26,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 568 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397386528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:26,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:26,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:26,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:26,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 569 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397386739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:26,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/59cae3d732d6462cadda6bc4523fc1bb 2024-12-05T11:15:26,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c17bb07a779f43fbb42e096d7bfc6d07 is 50, key is test_row_0/C:col10/1733397326224/Put/seqid=0 2024-12-05T11:15:26,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:15:26,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742109_1285 (size=12301) 2024-12-05T11:15:26,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:26,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 570 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397387049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:27,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:27,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c17bb07a779f43fbb42e096d7bfc6d07 2024-12-05T11:15:27,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/5154e89591934eae963e6d0f6b31c172 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/5154e89591934eae963e6d0f6b31c172 2024-12-05T11:15:27,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/5154e89591934eae963e6d0f6b31c172, entries=200, sequenceid=640, filesize=14.4 K 2024-12-05T11:15:27,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/59cae3d732d6462cadda6bc4523fc1bb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/59cae3d732d6462cadda6bc4523fc1bb 2024-12-05T11:15:27,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/59cae3d732d6462cadda6bc4523fc1bb, entries=150, sequenceid=640, filesize=12.0 K 2024-12-05T11:15:27,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c17bb07a779f43fbb42e096d7bfc6d07 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c17bb07a779f43fbb42e096d7bfc6d07 2024-12-05T11:15:27,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c17bb07a779f43fbb42e096d7bfc6d07, entries=150, sequenceid=640, filesize=12.0 K 2024-12-05T11:15:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 04e661eeba0fa942008ec581fcd28e79 in 1020ms, sequenceid=640, compaction requested=true 2024-12-05T11:15:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:27,246 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:27,246 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:27,248 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:27,248 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:27,248 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,248 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/21e056463f8e447c8812da4cf2deebd2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f0fef308140c404d97c96991a472c82b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/59cae3d732d6462cadda6bc4523fc1bb] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=36.7 K 2024-12-05T11:15:27,248 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:27,248 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:27,248 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,249 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/ebe97e243d77401f87621ab5107bc850, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/075ecba491244e149d12d9dbc9c2e8d3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/5154e89591934eae963e6d0f6b31c172] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=39.1 K 2024-12-05T11:15:27,249 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe97e243d77401f87621ab5107bc850, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733397322199 2024-12-05T11:15:27,250 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e056463f8e447c8812da4cf2deebd2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733397322199 2024-12-05T11:15:27,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T11:15:27,250 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f0fef308140c404d97c96991a472c82b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1733397324046 2024-12-05T11:15:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,250 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075ecba491244e149d12d9dbc9c2e8d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1733397324046 2024-12-05T11:15:27,251 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T11:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:27,251 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 59cae3d732d6462cadda6bc4523fc1bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733397326219 2024-12-05T11:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:27,251 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5154e89591934eae963e6d0f6b31c172, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733397326214 2024-12-05T11:15:27,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/58b57789388b40e2877d7f96daee0e2b is 50, key is test_row_0/A:col10/1733397326410/Put/seqid=0 2024-12-05T11:15:27,266 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:27,266 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:27,267 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4fbac935287d4390ab36875ebf100a09 is 50, key is test_row_0/A:col10/1733397326224/Put/seqid=0 2024-12-05T11:15:27,267 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/4672c3324b354d178692d853e68f10e6 is 50, key is test_row_0/B:col10/1733397326224/Put/seqid=0 2024-12-05T11:15:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742110_1286 (size=12301) 2024-12-05T11:15:27,281 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/58b57789388b40e2877d7f96daee0e2b 2024-12-05T11:15:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742111_1287 (size=13051) 2024-12-05T11:15:27,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742112_1288 (size=13051) 2024-12-05T11:15:27,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/4f2584e593f14e4586e83282a5af470a is 50, key is test_row_0/B:col10/1733397326410/Put/seqid=0 2024-12-05T11:15:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:15:27,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742113_1289 (size=12301) 2024-12-05T11:15:27,323 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/4f2584e593f14e4586e83282a5af470a 2024-12-05T11:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c170ce147acb4ff58f673a4fe7bc6afd is 50, key is test_row_0/C:col10/1733397326410/Put/seqid=0 2024-12-05T11:15:27,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742114_1290 (size=12301) 2024-12-05T11:15:27,370 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c170ce147acb4ff58f673a4fe7bc6afd 2024-12-05T11:15:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/58b57789388b40e2877d7f96daee0e2b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/58b57789388b40e2877d7f96daee0e2b 2024-12-05T11:15:27,383 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/58b57789388b40e2877d7f96daee0e2b, entries=150, sequenceid=677, filesize=12.0 K 2024-12-05T11:15:27,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/4f2584e593f14e4586e83282a5af470a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4f2584e593f14e4586e83282a5af470a 2024-12-05T11:15:27,388 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4f2584e593f14e4586e83282a5af470a, entries=150, sequenceid=677, filesize=12.0 K 2024-12-05T11:15:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/c170ce147acb4ff58f673a4fe7bc6afd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c170ce147acb4ff58f673a4fe7bc6afd 2024-12-05T11:15:27,395 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c170ce147acb4ff58f673a4fe7bc6afd, entries=150, sequenceid=677, filesize=12.0 K 2024-12-05T11:15:27,396 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 04e661eeba0fa942008ec581fcd28e79 in 146ms, sequenceid=677, compaction requested=true 2024-12-05T11:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-05T11:15:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-05T11:15:27,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T11:15:27,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2310 sec 2024-12-05T11:15:27,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2380 sec 2024-12-05T11:15:27,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:15:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:27,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:27,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/1ec486fe95ab4347bab6d8693051e3f3 is 50, key is test_row_0/A:col10/1733397327574/Put/seqid=0 2024-12-05T11:15:27,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742115_1291 (size=14741) 2024-12-05T11:15:27,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=687 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/1ec486fe95ab4347bab6d8693051e3f3 2024-12-05T11:15:27,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/5d5169c4fa424a68abd54639c24f5d02 is 50, key is test_row_0/B:col10/1733397327574/Put/seqid=0 2024-12-05T11:15:27,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742116_1292 (size=12301) 2024-12-05T11:15:27,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=687 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/5d5169c4fa424a68abd54639c24f5d02 2024-12-05T11:15:27,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/ee354bb053fc41a6bc70bbcd1d032c82 is 50, key is test_row_0/C:col10/1733397327574/Put/seqid=0 2024-12-05T11:15:27,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397327637, max timestamp=1733397327658Num uniques 859; 2024-12-05T11:15:27,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:27,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397327637, max timestamp=1733397327658Num uniques 859; 2024-12-05T11:15:27,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:27,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397327637, max timestamp=1733397327658Num uniques 859; 2024-12-05T11:15:27,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:27,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742117_1293 (size=12301) 2024-12-05T11:15:27,690 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/4fbac935287d4390ab36875ebf100a09 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4fbac935287d4390ab36875ebf100a09 2024-12-05T11:15:27,692 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/4672c3324b354d178692d853e68f10e6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4672c3324b354d178692d853e68f10e6 2024-12-05T11:15:27,696 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into 4fbac935287d4390ab36875ebf100a09(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:27,696 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:27,697 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397327246; duration=0sec 2024-12-05T11:15:27,697 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:27,697 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:27,697 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:27,698 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:27,698 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:27,698 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:27,698 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/00201c8931d54e8989156bc7776cde60, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/1f40380bbdf74dea88da85a704578856, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c17bb07a779f43fbb42e096d7bfc6d07, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c170ce147acb4ff58f673a4fe7bc6afd] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=48.7 K 2024-12-05T11:15:27,699 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00201c8931d54e8989156bc7776cde60, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1733397322199 2024-12-05T11:15:27,699 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into 4672c3324b354d178692d853e68f10e6(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:27,699 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:27,699 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397327246; duration=0sec 2024-12-05T11:15:27,699 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:27,699 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:27,700 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f40380bbdf74dea88da85a704578856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1733397324046 2024-12-05T11:15:27,700 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c17bb07a779f43fbb42e096d7bfc6d07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733397326219 2024-12-05T11:15:27,701 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c170ce147acb4ff58f673a4fe7bc6afd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1733397326393 2024-12-05T11:15:27,710 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:27,710 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/cc5f7d957b754129b4ebaea771687571 is 50, key is test_row_0/C:col10/1733397326410/Put/seqid=0 2024-12-05T11:15:27,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742118_1294 (size=13085) 2024-12-05T11:15:27,725 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/cc5f7d957b754129b4ebaea771687571 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/cc5f7d957b754129b4ebaea771687571 2024-12-05T11:15:27,732 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into cc5f7d957b754129b4ebaea771687571(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:27,732 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:27,732 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=12, startTime=1733397327247; duration=0sec 2024-12-05T11:15:27,733 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:27,733 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:27,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 612 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397387751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 613 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397387858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:28,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=687 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/ee354bb053fc41a6bc70bbcd1d032c82 2024-12-05T11:15:28,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/1ec486fe95ab4347bab6d8693051e3f3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/1ec486fe95ab4347bab6d8693051e3f3 2024-12-05T11:15:28,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 614 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397388068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:28,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/1ec486fe95ab4347bab6d8693051e3f3, entries=200, sequenceid=687, filesize=14.4 K 2024-12-05T11:15:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/5d5169c4fa424a68abd54639c24f5d02 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/5d5169c4fa424a68abd54639c24f5d02 2024-12-05T11:15:28,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/5d5169c4fa424a68abd54639c24f5d02, entries=150, sequenceid=687, filesize=12.0 K 2024-12-05T11:15:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/ee354bb053fc41a6bc70bbcd1d032c82 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/ee354bb053fc41a6bc70bbcd1d032c82 2024-12-05T11:15:28,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/ee354bb053fc41a6bc70bbcd1d032c82, entries=150, sequenceid=687, filesize=12.0 K 2024-12-05T11:15:28,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 04e661eeba0fa942008ec581fcd28e79 in 501ms, sequenceid=687, compaction requested=true 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:28,080 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:28,080 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:28,081 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:28,081 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/A is initiating minor compaction (all files) 2024-12-05T11:15:28,081 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/A in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,081 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4fbac935287d4390ab36875ebf100a09, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/58b57789388b40e2877d7f96daee0e2b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/1ec486fe95ab4347bab6d8693051e3f3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=39.2 K 2024-12-05T11:15:28,081 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:28,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/B is initiating minor compaction (all files) 2024-12-05T11:15:28,082 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/B in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,082 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbac935287d4390ab36875ebf100a09, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733397326219 2024-12-05T11:15:28,082 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4672c3324b354d178692d853e68f10e6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4f2584e593f14e4586e83282a5af470a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/5d5169c4fa424a68abd54639c24f5d02] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=36.8 K 2024-12-05T11:15:28,082 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b57789388b40e2877d7f96daee0e2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1733397326393 2024-12-05T11:15:28,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 4672c3324b354d178692d853e68f10e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733397326219 2024-12-05T11:15:28,082 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec486fe95ab4347bab6d8693051e3f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733397327568 2024-12-05T11:15:28,082 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2584e593f14e4586e83282a5af470a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1733397326393 2024-12-05T11:15:28,083 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d5169c4fa424a68abd54639c24f5d02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733397327568 2024-12-05T11:15:28,089 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:28,089 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:28,090 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/3ec7cee036f1460a88cefa22904b38e5 is 50, key is test_row_0/A:col10/1733397327574/Put/seqid=0 2024-12-05T11:15:28,090 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/ae4f6cb77f8446b5a164c612999bf8d4 is 50, key is test_row_0/B:col10/1733397327574/Put/seqid=0 2024-12-05T11:15:28,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742119_1295 (size=13153) 2024-12-05T11:15:28,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742120_1296 (size=13153) 2024-12-05T11:15:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T11:15:28,309 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:28,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-05T11:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:28,313 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:28,313 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:28,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:28,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-05T11:15:28,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:28,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:15:28,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:28,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:15:28,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:28,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:15:28,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/7a33d6ab112b4cda802d18260501a26b is 50, key is test_row_0/A:col10/1733397327746/Put/seqid=0 2024-12-05T11:15:28,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742121_1297 (size=12301) 2024-12-05T11:15:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:28,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 622 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397388421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:28,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,526 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/3ec7cee036f1460a88cefa22904b38e5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3ec7cee036f1460a88cefa22904b38e5 2024-12-05T11:15:28,530 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/ae4f6cb77f8446b5a164c612999bf8d4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/ae4f6cb77f8446b5a164c612999bf8d4 2024-12-05T11:15:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 623 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397388528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:28,538 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/A of 04e661eeba0fa942008ec581fcd28e79 into 3ec7cee036f1460a88cefa22904b38e5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:28,538 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:28,538 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/A, priority=13, startTime=1733397328080; duration=0sec 2024-12-05T11:15:28,538 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:28,538 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:28,538 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T11:15:28,542 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:15:28,543 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:15:28,543 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/B of 04e661eeba0fa942008ec581fcd28e79 into ae4f6cb77f8446b5a164c612999bf8d4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:28,543 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:28,543 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/B, priority=13, startTime=1733397328080; duration=0sec 2024-12-05T11:15:28,543 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:28,543 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:28,544 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. because compaction request was cancelled 2024-12-05T11:15:28,544 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:28,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:28,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 624 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397388738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:28,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/7a33d6ab112b4cda802d18260501a26b 2024-12-05T11:15:28,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/413eec1a0707498e80e831174cf28516 is 50, key is test_row_0/B:col10/1733397327746/Put/seqid=0 2024-12-05T11:15:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742122_1298 (size=12301) 2024-12-05T11:15:28,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:28,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:28,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:28,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:29,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 625 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397389049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:29,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/413eec1a0707498e80e831174cf28516 2024-12-05T11:15:29,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/86cd62fbaeb64cf9b41a8c12d5074b51 is 50, key is test_row_0/C:col10/1733397327746/Put/seqid=0 2024-12-05T11:15:29,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742123_1299 (size=12301) 2024-12-05T11:15:29,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:29,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:29,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:29,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 626 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58340 deadline: 1733397389558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:15:29,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/86cd62fbaeb64cf9b41a8c12d5074b51 2024-12-05T11:15:29,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/7a33d6ab112b4cda802d18260501a26b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/7a33d6ab112b4cda802d18260501a26b 2024-12-05T11:15:29,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/7a33d6ab112b4cda802d18260501a26b, entries=150, sequenceid=728, filesize=12.0 K 2024-12-05T11:15:29,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/413eec1a0707498e80e831174cf28516 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/413eec1a0707498e80e831174cf28516 2024-12-05T11:15:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:29,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/413eec1a0707498e80e831174cf28516, entries=150, sequenceid=728, filesize=12.0 K 2024-12-05T11:15:29,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/86cd62fbaeb64cf9b41a8c12d5074b51 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86cd62fbaeb64cf9b41a8c12d5074b51 2024-12-05T11:15:29,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86cd62fbaeb64cf9b41a8c12d5074b51, entries=150, sequenceid=728, filesize=12.0 K 2024-12-05T11:15:29,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 04e661eeba0fa942008ec581fcd28e79 in 1328ms, sequenceid=728, compaction requested=true 2024-12-05T11:15:29,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T11:15:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T11:15:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04e661eeba0fa942008ec581fcd28e79:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. because compaction request was cancelled 2024-12-05T11:15:29,710 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. because compaction request was cancelled 2024-12-05T11:15:29,711 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:B 2024-12-05T11:15:29,711 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:A 2024-12-05T11:15:29,711 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:29,712 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:29,712 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): 04e661eeba0fa942008ec581fcd28e79/C is initiating minor compaction (all files) 2024-12-05T11:15:29,712 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04e661eeba0fa942008ec581fcd28e79/C in TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,712 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/cc5f7d957b754129b4ebaea771687571, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/ee354bb053fc41a6bc70bbcd1d032c82, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86cd62fbaeb64cf9b41a8c12d5074b51] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp, totalSize=36.8 K 2024-12-05T11:15:29,713 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5f7d957b754129b4ebaea771687571, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1733397326393 2024-12-05T11:15:29,713 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee354bb053fc41a6bc70bbcd1d032c82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=687, earliestPutTs=1733397327568 2024-12-05T11:15:29,713 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86cd62fbaeb64cf9b41a8c12d5074b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733397327746 2024-12-05T11:15:29,724 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04e661eeba0fa942008ec581fcd28e79#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:29,726 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/0cc18299c7234344b82fa0147db43684 is 50, key is test_row_0/C:col10/1733397327746/Put/seqid=0 2024-12-05T11:15:29,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742124_1300 (size=13187) 2024-12-05T11:15:29,749 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/0cc18299c7234344b82fa0147db43684 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/0cc18299c7234344b82fa0147db43684 2024-12-05T11:15:29,768 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04e661eeba0fa942008ec581fcd28e79/C of 04e661eeba0fa942008ec581fcd28e79 into 0cc18299c7234344b82fa0147db43684(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:29,768 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:29,768 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., storeName=04e661eeba0fa942008ec581fcd28e79/C, priority=13, startTime=1733397329710; duration=0sec 2024-12-05T11:15:29,769 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:29,769 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04e661eeba0fa942008ec581fcd28e79:C 2024-12-05T11:15:29,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:29,850 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:29,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/28a29809d0ff45e2b09d107ee860d28b is 50, key is test_row_0/A:col10/1733397328414/Put/seqid=0 2024-12-05T11:15:29,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742125_1301 (size=12301) 2024-12-05T11:15:30,235 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-12-05T11:15:30,235 INFO [Thread-1046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1046. 2024-12-05T11:15:30,235 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:30,235 DEBUG [Thread-1046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:30,235 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,235 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,235 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,235 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:30,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:30,237 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-12-05T11:15:30,237 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:30,237 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,237 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:30,238 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-12-05T11:15:30,238 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-12-05T11:15:30,238 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:30,238 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,238 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:30,238 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,238 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,238 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:30,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:30,267 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/28a29809d0ff45e2b09d107ee860d28b 2024-12-05T11:15:30,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/33d3f44ad6d7447ea8fa965f2822e993 is 50, key is test_row_0/B:col10/1733397328414/Put/seqid=0 2024-12-05T11:15:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742126_1302 (size=12301) 2024-12-05T11:15:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. as already flushing 2024-12-05T11:15:30,579 INFO [Thread-1037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1037. 2024-12-05T11:15:30,579 DEBUG [Thread-1037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:30,579 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,579 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:30,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:30,678 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/33d3f44ad6d7447ea8fa965f2822e993 2024-12-05T11:15:30,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/e6ce0b9d9e3a4155a875c24c32a87623 is 50, key is test_row_0/C:col10/1733397328414/Put/seqid=0 2024-12-05T11:15:30,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742127_1303 (size=12301) 2024-12-05T11:15:31,090 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/e6ce0b9d9e3a4155a875c24c32a87623 2024-12-05T11:15:31,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/28a29809d0ff45e2b09d107ee860d28b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/28a29809d0ff45e2b09d107ee860d28b 2024-12-05T11:15:31,097 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/28a29809d0ff45e2b09d107ee860d28b, entries=150, sequenceid=740, filesize=12.0 K 2024-12-05T11:15:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/33d3f44ad6d7447ea8fa965f2822e993 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/33d3f44ad6d7447ea8fa965f2822e993 2024-12-05T11:15:31,101 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/33d3f44ad6d7447ea8fa965f2822e993, entries=150, sequenceid=740, filesize=12.0 K 2024-12-05T11:15:31,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/e6ce0b9d9e3a4155a875c24c32a87623 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/e6ce0b9d9e3a4155a875c24c32a87623 2024-12-05T11:15:31,105 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/e6ce0b9d9e3a4155a875c24c32a87623, entries=150, sequenceid=740, filesize=12.0 K 2024-12-05T11:15:31,105 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for 04e661eeba0fa942008ec581fcd28e79 in 1255ms, sequenceid=740, compaction requested=true 2024-12-05T11:15:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 04e661eeba0fa942008ec581fcd28e79: 2024-12-05T11:15:31,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:31,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-05T11:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-05T11:15:31,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T11:15:31,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7940 sec 2024-12-05T11:15:31,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.7990 sec 2024-12-05T11:15:32,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:15:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T11:15:32,478 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:32,772 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-12-05T11:15:32,772 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:32,772 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,772 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:32,809 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-12-05T11:15:32,809 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:32,809 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,809 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:32,830 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-12-05T11:15:32,830 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:32,830 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,830 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:32,880 INFO [Thread-1035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1035. 2024-12-05T11:15:32,880 DEBUG [Thread-1035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:32,880 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,880 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 576 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-05T11:15:32,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:15:32,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4451 2024-12-05T11:15:32,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4359 2024-12-05T11:15:32,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4264 2024-12-05T11:15:32,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4476 2024-12-05T11:15:32,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4352 2024-12-05T11:15:32,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:15:32,881 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:15:32,881 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:15:32,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:32,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:32,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:15:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:15:32,888 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397332887"}]},"ts":"1733397332887"} 2024-12-05T11:15:32,890 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:15:32,890 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:15:32,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:15:32,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, UNASSIGN}] 2024-12-05T11:15:32,894 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, UNASSIGN 2024-12-05T11:15:32,895 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=04e661eeba0fa942008ec581fcd28e79, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:32,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, UNASSIGN because future has completed 2024-12-05T11:15:32,896 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:15:32,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:15:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:15:33,049 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:33,049 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:15:33,049 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing 04e661eeba0fa942008ec581fcd28e79, disabling compactions & flushes 2024-12-05T11:15:33,049 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:33,049 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:33,049 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. after waiting 0 ms 2024-12-05T11:15:33,049 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:33,049 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing 04e661eeba0fa942008ec581fcd28e79 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T11:15:33,050 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=A 2024-12-05T11:15:33,050 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:33,050 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=B 2024-12-05T11:15:33,050 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:33,050 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04e661eeba0fa942008ec581fcd28e79, store=C 2024-12-05T11:15:33,050 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:33,053 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/123c918b87b14178b2c4bdbc7bb380e3 is 50, key is test_row_0/A:col10/1733397332878/Put/seqid=0 2024-12-05T11:15:33,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742128_1304 (size=9857) 2024-12-05T11:15:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:15:33,458 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/123c918b87b14178b2c4bdbc7bb380e3 2024-12-05T11:15:33,472 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/20c75ed6760141a4bdf7068a37edb08b is 50, key is test_row_0/B:col10/1733397332878/Put/seqid=0 2024-12-05T11:15:33,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742129_1305 (size=9857) 2024-12-05T11:15:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:15:33,879 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/20c75ed6760141a4bdf7068a37edb08b 2024-12-05T11:15:33,893 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/a715d02f798b49dd8956133c666fbbe2 is 50, key is test_row_0/C:col10/1733397332878/Put/seqid=0 2024-12-05T11:15:33,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742130_1306 (size=9857) 2024-12-05T11:15:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:15:34,300 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/a715d02f798b49dd8956133c666fbbe2 2024-12-05T11:15:34,312 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/A/123c918b87b14178b2c4bdbc7bb380e3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/123c918b87b14178b2c4bdbc7bb380e3 2024-12-05T11:15:34,320 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/123c918b87b14178b2c4bdbc7bb380e3, entries=100, sequenceid=748, filesize=9.6 K 2024-12-05T11:15:34,321 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/B/20c75ed6760141a4bdf7068a37edb08b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/20c75ed6760141a4bdf7068a37edb08b 2024-12-05T11:15:34,326 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/20c75ed6760141a4bdf7068a37edb08b, entries=100, sequenceid=748, filesize=9.6 K 2024-12-05T11:15:34,326 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/.tmp/C/a715d02f798b49dd8956133c666fbbe2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a715d02f798b49dd8956133c666fbbe2 2024-12-05T11:15:34,332 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a715d02f798b49dd8956133c666fbbe2, entries=100, sequenceid=748, filesize=9.6 K 2024-12-05T11:15:34,333 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 04e661eeba0fa942008ec581fcd28e79 in 1284ms, sequenceid=748, compaction requested=true 2024-12-05T11:15:34,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/e576f13a197b48d0b4665b555d23f535, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8e666398a9314713b96fe31eb4ec3e3f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/bacc119f5a1f42a480fed2eba2ee102c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/11d6b5652a2a4d6caf07a66da2ee485b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4d5d5002795d48769ff5de7307568932, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/0100bb40ad5a4b0cb66528e07279892d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/b93f72aa1b934540b4f6f44945f0ab4f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/6e19818b684a40e29b919dcc28cb8177, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/550abafa1884492d844d63eb18285e7c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8d457c797f6a4d16a3b050c2ced9a4ce, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/181551588ce54aef8c3fc0a61e12440f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/85e97ce899534a999addfaf45f840f11, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/66a3b24ac51b4e08860220168d96c4d1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8a8c4870914148b1b089ec62c735aae3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/57278e2e7a6d43908dc64ade87620a05, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/56157d7f9f94493d8107094593b184e4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/eb8bda54ab0b4e9f8b93d0743b719848, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4a7faf63c75e4d0c9e9092f92dc43f3e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/ebe97e243d77401f87621ab5107bc850, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3755a53ec71a441c91a29af16619fd03, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/075ecba491244e149d12d9dbc9c2e8d3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/5154e89591934eae963e6d0f6b31c172, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4fbac935287d4390ab36875ebf100a09, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/58b57789388b40e2877d7f96daee0e2b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/1ec486fe95ab4347bab6d8693051e3f3] to archive 2024-12-05T11:15:34,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:15:34,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/e576f13a197b48d0b4665b555d23f535 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/e576f13a197b48d0b4665b555d23f535 2024-12-05T11:15:34,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8e666398a9314713b96fe31eb4ec3e3f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8e666398a9314713b96fe31eb4ec3e3f 2024-12-05T11:15:34,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/bacc119f5a1f42a480fed2eba2ee102c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/bacc119f5a1f42a480fed2eba2ee102c 2024-12-05T11:15:34,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/11d6b5652a2a4d6caf07a66da2ee485b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/11d6b5652a2a4d6caf07a66da2ee485b 2024-12-05T11:15:34,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4d5d5002795d48769ff5de7307568932 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4d5d5002795d48769ff5de7307568932 2024-12-05T11:15:34,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/0100bb40ad5a4b0cb66528e07279892d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/0100bb40ad5a4b0cb66528e07279892d 2024-12-05T11:15:34,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/b93f72aa1b934540b4f6f44945f0ab4f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/b93f72aa1b934540b4f6f44945f0ab4f 2024-12-05T11:15:34,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/6e19818b684a40e29b919dcc28cb8177 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/6e19818b684a40e29b919dcc28cb8177 2024-12-05T11:15:34,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/550abafa1884492d844d63eb18285e7c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/550abafa1884492d844d63eb18285e7c 2024-12-05T11:15:34,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8d457c797f6a4d16a3b050c2ced9a4ce to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8d457c797f6a4d16a3b050c2ced9a4ce 2024-12-05T11:15:34,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/181551588ce54aef8c3fc0a61e12440f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/181551588ce54aef8c3fc0a61e12440f 2024-12-05T11:15:34,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/85e97ce899534a999addfaf45f840f11 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/85e97ce899534a999addfaf45f840f11 2024-12-05T11:15:34,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/66a3b24ac51b4e08860220168d96c4d1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/66a3b24ac51b4e08860220168d96c4d1 2024-12-05T11:15:34,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8a8c4870914148b1b089ec62c735aae3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/8a8c4870914148b1b089ec62c735aae3 2024-12-05T11:15:34,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/57278e2e7a6d43908dc64ade87620a05 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/57278e2e7a6d43908dc64ade87620a05 2024-12-05T11:15:34,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/56157d7f9f94493d8107094593b184e4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/56157d7f9f94493d8107094593b184e4 2024-12-05T11:15:34,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/eb8bda54ab0b4e9f8b93d0743b719848 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/eb8bda54ab0b4e9f8b93d0743b719848 2024-12-05T11:15:34,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4a7faf63c75e4d0c9e9092f92dc43f3e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4a7faf63c75e4d0c9e9092f92dc43f3e 2024-12-05T11:15:34,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/ebe97e243d77401f87621ab5107bc850 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/ebe97e243d77401f87621ab5107bc850 2024-12-05T11:15:34,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3755a53ec71a441c91a29af16619fd03 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3755a53ec71a441c91a29af16619fd03 2024-12-05T11:15:34,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/075ecba491244e149d12d9dbc9c2e8d3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/075ecba491244e149d12d9dbc9c2e8d3 2024-12-05T11:15:34,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/5154e89591934eae963e6d0f6b31c172 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/5154e89591934eae963e6d0f6b31c172 2024-12-05T11:15:34,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4fbac935287d4390ab36875ebf100a09 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/4fbac935287d4390ab36875ebf100a09 2024-12-05T11:15:34,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/58b57789388b40e2877d7f96daee0e2b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/58b57789388b40e2877d7f96daee0e2b 2024-12-05T11:15:34,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/1ec486fe95ab4347bab6d8693051e3f3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/1ec486fe95ab4347bab6d8693051e3f3 2024-12-05T11:15:34,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/11be532676324a039cca2577674245f1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d202b45c691c499fbfeb364ce685cf5e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/995e8da5f2a14f43be0b1a0be80444dd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/a9cef8039dc945fb9fd961e14df99f6d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/85400207bbc5462783dd5d073223bd08, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e11b23fba1e457bb829daf89a40598e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/482b4be21cc345d0bf92792783f39109, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/0ab5a352e1944612be06d2ffd096da68, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d1aa0974478d48869e96d83fb8d9fe90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/544586ff90e8426d843554a2af01f497, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e0a8b9788844ee6b70b3d6ef4a5a7de, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/abb619744605459eb53fe5b7dd91ff69, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/88aadded26b44551b04a3fdeab424d0c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f7997c78d0a04edaa029dddb96c99736, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/b088cf3ca7494416a40a684beeb9b1d2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/7627b99febbe40418d9aff6045caca8c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/871f9f2e59ba483083d07b70a11ef393, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fddf734351f4306ba30bb2e17033434, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/21e056463f8e447c8812da4cf2deebd2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fc96ce124024d6d97952b65c98bcde1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f0fef308140c404d97c96991a472c82b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4672c3324b354d178692d853e68f10e6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/59cae3d732d6462cadda6bc4523fc1bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4f2584e593f14e4586e83282a5af470a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/5d5169c4fa424a68abd54639c24f5d02] to archive 2024-12-05T11:15:34,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:15:34,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/11be532676324a039cca2577674245f1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/11be532676324a039cca2577674245f1 2024-12-05T11:15:34,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d202b45c691c499fbfeb364ce685cf5e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d202b45c691c499fbfeb364ce685cf5e 2024-12-05T11:15:34,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/995e8da5f2a14f43be0b1a0be80444dd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/995e8da5f2a14f43be0b1a0be80444dd 2024-12-05T11:15:34,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/a9cef8039dc945fb9fd961e14df99f6d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/a9cef8039dc945fb9fd961e14df99f6d 2024-12-05T11:15:34,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/85400207bbc5462783dd5d073223bd08 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/85400207bbc5462783dd5d073223bd08 2024-12-05T11:15:34,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e11b23fba1e457bb829daf89a40598e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e11b23fba1e457bb829daf89a40598e 2024-12-05T11:15:34,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/482b4be21cc345d0bf92792783f39109 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/482b4be21cc345d0bf92792783f39109 2024-12-05T11:15:34,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/0ab5a352e1944612be06d2ffd096da68 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/0ab5a352e1944612be06d2ffd096da68 2024-12-05T11:15:34,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d1aa0974478d48869e96d83fb8d9fe90 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/d1aa0974478d48869e96d83fb8d9fe90 2024-12-05T11:15:34,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/544586ff90e8426d843554a2af01f497 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/544586ff90e8426d843554a2af01f497 2024-12-05T11:15:34,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e0a8b9788844ee6b70b3d6ef4a5a7de to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1e0a8b9788844ee6b70b3d6ef4a5a7de 2024-12-05T11:15:34,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/abb619744605459eb53fe5b7dd91ff69 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/abb619744605459eb53fe5b7dd91ff69 2024-12-05T11:15:34,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/88aadded26b44551b04a3fdeab424d0c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/88aadded26b44551b04a3fdeab424d0c 2024-12-05T11:15:34,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f7997c78d0a04edaa029dddb96c99736 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f7997c78d0a04edaa029dddb96c99736 2024-12-05T11:15:34,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/b088cf3ca7494416a40a684beeb9b1d2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/b088cf3ca7494416a40a684beeb9b1d2 2024-12-05T11:15:34,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/7627b99febbe40418d9aff6045caca8c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/7627b99febbe40418d9aff6045caca8c 2024-12-05T11:15:34,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/871f9f2e59ba483083d07b70a11ef393 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/871f9f2e59ba483083d07b70a11ef393 2024-12-05T11:15:34,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fddf734351f4306ba30bb2e17033434 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fddf734351f4306ba30bb2e17033434 2024-12-05T11:15:34,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/21e056463f8e447c8812da4cf2deebd2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/21e056463f8e447c8812da4cf2deebd2 2024-12-05T11:15:34,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fc96ce124024d6d97952b65c98bcde1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/1fc96ce124024d6d97952b65c98bcde1 2024-12-05T11:15:34,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f0fef308140c404d97c96991a472c82b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/f0fef308140c404d97c96991a472c82b 2024-12-05T11:15:34,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4672c3324b354d178692d853e68f10e6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4672c3324b354d178692d853e68f10e6 2024-12-05T11:15:34,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/59cae3d732d6462cadda6bc4523fc1bb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/59cae3d732d6462cadda6bc4523fc1bb 2024-12-05T11:15:34,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4f2584e593f14e4586e83282a5af470a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/4f2584e593f14e4586e83282a5af470a 2024-12-05T11:15:34,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/5d5169c4fa424a68abd54639c24f5d02 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/5d5169c4fa424a68abd54639c24f5d02 2024-12-05T11:15:34,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/7d06e70ea99e4835af72b467c15b3dbd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/244150eef7ce4d81a854dc278a6c6133, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a36e1a62446e40efafa06b753db63bd3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4ffccbf60f4340da9d8be00a215c884e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/9703d158aa7948d7aae9aed2aeba6e73, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/2abb717aef5f4dbf9aaa737d5682ec90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/b8646d0109e049dca16083eb7fcc9a37, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d33bd49de5324f98b75a2cbbd931f7e5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c382bf6556924652b4621c53ce7ca7b0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/dea10971b1794ecaaa21c83ba5267c15, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d4a22baad1aa4698b047aefba38e8fca, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/61c6016d81b2405591a8cc3eacbab2c7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/74ecb637795b4c528287d68fa35979bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4c67dc1dd9a641f88541cd3a967ed114, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c3534e19c3724cbd954e13f1112f9f91, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/3012a7aa2c744ed4beca5525b4dafa6b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86ed3e56f5de423aa8f641847aa5b2fb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/8daada6be2ce451a9900059d224fccf7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/00201c8931d54e8989156bc7776cde60, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/37424dd030d44598846e56764f8b0ac7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/1f40380bbdf74dea88da85a704578856, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c17bb07a779f43fbb42e096d7bfc6d07, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/cc5f7d957b754129b4ebaea771687571, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c170ce147acb4ff58f673a4fe7bc6afd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/ee354bb053fc41a6bc70bbcd1d032c82, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86cd62fbaeb64cf9b41a8c12d5074b51] to archive 2024-12-05T11:15:34,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:15:34,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/7d06e70ea99e4835af72b467c15b3dbd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/7d06e70ea99e4835af72b467c15b3dbd 2024-12-05T11:15:34,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/244150eef7ce4d81a854dc278a6c6133 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/244150eef7ce4d81a854dc278a6c6133 2024-12-05T11:15:34,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a36e1a62446e40efafa06b753db63bd3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a36e1a62446e40efafa06b753db63bd3 2024-12-05T11:15:34,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4ffccbf60f4340da9d8be00a215c884e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4ffccbf60f4340da9d8be00a215c884e 2024-12-05T11:15:34,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/9703d158aa7948d7aae9aed2aeba6e73 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/9703d158aa7948d7aae9aed2aeba6e73 2024-12-05T11:15:34,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/2abb717aef5f4dbf9aaa737d5682ec90 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/2abb717aef5f4dbf9aaa737d5682ec90 2024-12-05T11:15:34,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/b8646d0109e049dca16083eb7fcc9a37 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/b8646d0109e049dca16083eb7fcc9a37 2024-12-05T11:15:34,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d33bd49de5324f98b75a2cbbd931f7e5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d33bd49de5324f98b75a2cbbd931f7e5 2024-12-05T11:15:34,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c382bf6556924652b4621c53ce7ca7b0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c382bf6556924652b4621c53ce7ca7b0 2024-12-05T11:15:34,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/dea10971b1794ecaaa21c83ba5267c15 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/dea10971b1794ecaaa21c83ba5267c15 2024-12-05T11:15:34,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d4a22baad1aa4698b047aefba38e8fca to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/d4a22baad1aa4698b047aefba38e8fca 2024-12-05T11:15:34,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/61c6016d81b2405591a8cc3eacbab2c7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/61c6016d81b2405591a8cc3eacbab2c7 2024-12-05T11:15:34,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/74ecb637795b4c528287d68fa35979bb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/74ecb637795b4c528287d68fa35979bb 2024-12-05T11:15:34,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4c67dc1dd9a641f88541cd3a967ed114 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/4c67dc1dd9a641f88541cd3a967ed114 2024-12-05T11:15:34,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c3534e19c3724cbd954e13f1112f9f91 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c3534e19c3724cbd954e13f1112f9f91 2024-12-05T11:15:34,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/3012a7aa2c744ed4beca5525b4dafa6b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/3012a7aa2c744ed4beca5525b4dafa6b 2024-12-05T11:15:34,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86ed3e56f5de423aa8f641847aa5b2fb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86ed3e56f5de423aa8f641847aa5b2fb 2024-12-05T11:15:34,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/8daada6be2ce451a9900059d224fccf7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/8daada6be2ce451a9900059d224fccf7 2024-12-05T11:15:34,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/00201c8931d54e8989156bc7776cde60 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/00201c8931d54e8989156bc7776cde60 2024-12-05T11:15:34,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/37424dd030d44598846e56764f8b0ac7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/37424dd030d44598846e56764f8b0ac7 2024-12-05T11:15:34,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/1f40380bbdf74dea88da85a704578856 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/1f40380bbdf74dea88da85a704578856 2024-12-05T11:15:34,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c17bb07a779f43fbb42e096d7bfc6d07 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c17bb07a779f43fbb42e096d7bfc6d07 2024-12-05T11:15:34,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/cc5f7d957b754129b4ebaea771687571 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/cc5f7d957b754129b4ebaea771687571 2024-12-05T11:15:34,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c170ce147acb4ff58f673a4fe7bc6afd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/c170ce147acb4ff58f673a4fe7bc6afd 2024-12-05T11:15:34,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/ee354bb053fc41a6bc70bbcd1d032c82 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/ee354bb053fc41a6bc70bbcd1d032c82 2024-12-05T11:15:34,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86cd62fbaeb64cf9b41a8c12d5074b51 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/86cd62fbaeb64cf9b41a8c12d5074b51 2024-12-05T11:15:34,445 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/recovered.edits/751.seqid, newMaxSeqId=751, maxSeqId=1 2024-12-05T11:15:34,446 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. 2024-12-05T11:15:34,446 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for 04e661eeba0fa942008ec581fcd28e79: Waiting for close lock at 1733397333049Running coprocessor pre-close hooks at 1733397333049Disabling compacts and flushes for region at 1733397333049Disabling writes for close at 1733397333049Obtaining lock to block concurrent updates at 1733397333049Preparing flush snapshotting stores in 04e661eeba0fa942008ec581fcd28e79 at 1733397333049Finished memstore snapshotting TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733397333050 (+1 ms)Flushing stores of TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79. at 1733397333050Flushing 04e661eeba0fa942008ec581fcd28e79/A: creating writer at 1733397333050Flushing 04e661eeba0fa942008ec581fcd28e79/A: appending metadata at 1733397333053 (+3 ms)Flushing 04e661eeba0fa942008ec581fcd28e79/A: closing flushed file at 1733397333053Flushing 04e661eeba0fa942008ec581fcd28e79/B: creating writer at 1733397333468 (+415 ms)Flushing 04e661eeba0fa942008ec581fcd28e79/B: appending metadata at 1733397333471 (+3 ms)Flushing 04e661eeba0fa942008ec581fcd28e79/B: closing flushed file at 1733397333472 (+1 ms)Flushing 04e661eeba0fa942008ec581fcd28e79/C: creating writer at 1733397333888 (+416 ms)Flushing 04e661eeba0fa942008ec581fcd28e79/C: appending metadata at 1733397333893 (+5 ms)Flushing 04e661eeba0fa942008ec581fcd28e79/C: closing flushed file at 1733397333893Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5363ae0a: reopening flushed file at 1733397334311 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be08b3b: reopening flushed file at 1733397334320 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c475000: reopening flushed file at 1733397334326 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 04e661eeba0fa942008ec581fcd28e79 in 1284ms, sequenceid=748, compaction requested=true at 1733397334333 (+7 ms)Writing region close event to WAL at 1733397334442 (+109 ms)Running coprocessor post-close hooks at 1733397334446 (+4 ms)Closed at 1733397334446 2024-12-05T11:15:34,448 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed 04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:34,448 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=04e661eeba0fa942008ec581fcd28e79, regionState=CLOSED 2024-12-05T11:15:34,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:15:34,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-05T11:15:34,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure 04e661eeba0fa942008ec581fcd28e79, server=6706a97ae65f,42931,1733397243639 in 1.5540 sec 2024-12-05T11:15:34,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-05T11:15:34,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04e661eeba0fa942008ec581fcd28e79, UNASSIGN in 1.5590 sec 2024-12-05T11:15:34,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-05T11:15:34,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5640 sec 2024-12-05T11:15:34,457 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397334457"}]},"ts":"1733397334457"} 2024-12-05T11:15:34,458 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:15:34,458 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:15:34,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5760 sec 2024-12-05T11:15:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T11:15:35,039 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:35,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:15:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,041 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:15:35,041 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,043 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:35,046 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/recovered.edits] 2024-12-05T11:15:35,047 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/123c918b87b14178b2c4bdbc7bb380e3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/123c918b87b14178b2c4bdbc7bb380e3 2024-12-05T11:15:35,049 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/28a29809d0ff45e2b09d107ee860d28b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/28a29809d0ff45e2b09d107ee860d28b 2024-12-05T11:15:35,050 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3ec7cee036f1460a88cefa22904b38e5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/3ec7cee036f1460a88cefa22904b38e5 2024-12-05T11:15:35,051 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/7a33d6ab112b4cda802d18260501a26b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/A/7a33d6ab112b4cda802d18260501a26b 2024-12-05T11:15:35,052 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/20c75ed6760141a4bdf7068a37edb08b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/20c75ed6760141a4bdf7068a37edb08b 2024-12-05T11:15:35,053 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/33d3f44ad6d7447ea8fa965f2822e993 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/33d3f44ad6d7447ea8fa965f2822e993 2024-12-05T11:15:35,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/413eec1a0707498e80e831174cf28516 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/413eec1a0707498e80e831174cf28516 2024-12-05T11:15:35,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/ae4f6cb77f8446b5a164c612999bf8d4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/B/ae4f6cb77f8446b5a164c612999bf8d4 2024-12-05T11:15:35,056 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/0cc18299c7234344b82fa0147db43684 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/0cc18299c7234344b82fa0147db43684 2024-12-05T11:15:35,057 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a715d02f798b49dd8956133c666fbbe2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/a715d02f798b49dd8956133c666fbbe2 2024-12-05T11:15:35,058 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/e6ce0b9d9e3a4155a875c24c32a87623 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/C/e6ce0b9d9e3a4155a875c24c32a87623 2024-12-05T11:15:35,060 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/recovered.edits/751.seqid to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79/recovered.edits/751.seqid 2024-12-05T11:15:35,061 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/04e661eeba0fa942008ec581fcd28e79 2024-12-05T11:15:35,061 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:15:35,063 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,065 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:15:35,067 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:15:35,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:15:35,068 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733397335068"}]},"ts":"9223372036854775807"} 2024-12-05T11:15:35,070 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:15:35,070 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 04e661eeba0fa942008ec581fcd28e79, NAME => 'TestAcidGuarantees,,1733397308025.04e661eeba0fa942008ec581fcd28e79.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:15:35,070 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:15:35,070 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733397335070"}]},"ts":"9223372036854775807"} 2024-12-05T11:15:35,072 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:15:35,073 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-12-05T11:15:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T11:15:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:15:35,149 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:35,158 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2084105885_22 at /127.0.0.1:49778 [Waiting for operation #309] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_711469046_22 at /127.0.0.1:48060 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=379 (was 335) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6796 (was 5982) - AvailableMemoryMB LEAK? - 2024-12-05T11:15:35,166 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=379, ProcessCount=11, AvailableMemoryMB=6794 2024-12-05T11:15:35,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:15:35,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:15:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:35,169 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:15:35,169 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:35,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-12-05T11:15:35,169 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:15:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-05T11:15:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742131_1307 (size=1061) 2024-12-05T11:15:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-05T11:15:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-05T11:15:35,579 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:15:35,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742132_1308 (size=53) 2024-12-05T11:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-05T11:15:35,989 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:15:35,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f72a0ecfe0b917c5e942c56777fb6984, disabling compactions & flushes 2024-12-05T11:15:35,990 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:35,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:35,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. after waiting 0 ms 2024-12-05T11:15:35,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:35,990 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:35,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f72a0ecfe0b917c5e942c56777fb6984: Waiting for close lock at 1733397335990Disabling compacts and flushes for region at 1733397335990Disabling writes for close at 1733397335990Writing region close event to WAL at 1733397335990Closed at 1733397335990 2024-12-05T11:15:35,991 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:15:35,991 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733397335991"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733397335991"}]},"ts":"1733397335991"} 2024-12-05T11:15:35,993 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:15:35,993 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:15:35,993 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397335993"}]},"ts":"1733397335993"} 2024-12-05T11:15:35,994 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:15:35,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, ASSIGN}] 2024-12-05T11:15:35,995 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, ASSIGN 2024-12-05T11:15:35,996 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, ASSIGN; state=OFFLINE, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=false 2024-12-05T11:15:36,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:36,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, ASSIGN because future has completed 2024-12-05T11:15:36,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:15:36,304 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:36,304 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:15:36,304 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,304 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:15:36,304 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,304 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,306 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,307 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:36,307 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72a0ecfe0b917c5e942c56777fb6984 columnFamilyName A 2024-12-05T11:15:36,307 DEBUG [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:36,308 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(327): Store=f72a0ecfe0b917c5e942c56777fb6984/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:36,308 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-05T11:15:36,309 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:36,309 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72a0ecfe0b917c5e942c56777fb6984 columnFamilyName B 2024-12-05T11:15:36,309 DEBUG [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:36,310 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(327): Store=f72a0ecfe0b917c5e942c56777fb6984/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:36,310 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,311 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:36,311 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72a0ecfe0b917c5e942c56777fb6984 columnFamilyName C 2024-12-05T11:15:36,311 DEBUG [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:36,311 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(327): Store=f72a0ecfe0b917c5e942c56777fb6984/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:36,311 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:36,311 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,312 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,313 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,314 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,314 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,314 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:15:36,316 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,318 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:15:36,319 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened f72a0ecfe0b917c5e942c56777fb6984; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63966288, jitterRate=-0.04682803153991699}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:15:36,319 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:36,320 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for f72a0ecfe0b917c5e942c56777fb6984: Running coprocessor pre-open hook at 1733397336305Writing region info on filesystem at 1733397336305Initializing all the Stores at 1733397336305Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397336305Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397336306 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397336306Cleaning up temporary data from old regions at 1733397336314 (+8 ms)Running coprocessor post-open hooks at 1733397336319 (+5 ms)Region opened successfully at 1733397336320 (+1 ms) 2024-12-05T11:15:36,320 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., pid=79, masterSystemTime=1733397336301 2024-12-05T11:15:36,322 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:36,322 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:36,323 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:36,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:15:36,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-05T11:15:36,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 in 176 msec 2024-12-05T11:15:36,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-05T11:15:36,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, ASSIGN in 331 msec 2024-12-05T11:15:36,329 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:15:36,329 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397336329"}]},"ts":"1733397336329"} 2024-12-05T11:15:36,331 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:15:36,331 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:15:36,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-12-05T11:15:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-05T11:15:37,318 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:37,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a859e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:37,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:37,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:37,320 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f71598d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:37,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f1385d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:37,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:37,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:37,324 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:15:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:15:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1839ef63 2024-12-05T11:15:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:15:37,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:15:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:15:37,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:15:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T11:15:37,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742133_1309 (size=1097) 2024-12-05T11:15:37,742 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T11:15:37,742 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T11:15:37,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:15:37,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, REOPEN/MOVE}] 2024-12-05T11:15:37,751 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, REOPEN/MOVE 2024-12-05T11:15:37,751 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:37,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, REOPEN/MOVE because future has completed 2024-12-05T11:15:37,753 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:15:37,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:15:37,905 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:37,905 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:15:37,905 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing f72a0ecfe0b917c5e942c56777fb6984, disabling compactions & flushes 2024-12-05T11:15:37,905 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:37,906 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:37,906 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. after waiting 0 ms 2024-12-05T11:15:37,906 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:37,912 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T11:15:37,914 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:37,914 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for f72a0ecfe0b917c5e942c56777fb6984: Waiting for close lock at 1733397337905Running coprocessor pre-close hooks at 1733397337905Disabling compacts and flushes for region at 1733397337905Disabling writes for close at 1733397337906 (+1 ms)Writing region close event to WAL at 1733397337906Running coprocessor post-close hooks at 1733397337913 (+7 ms)Closed at 1733397337913 2024-12-05T11:15:37,914 WARN [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: f72a0ecfe0b917c5e942c56777fb6984 to self. 2024-12-05T11:15:37,917 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:37,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=CLOSED 2024-12-05T11:15:37,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:15:37,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-05T11:15:37,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 in 169 msec 2024-12-05T11:15:37,926 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, REOPEN/MOVE; state=CLOSED, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=true 2024-12-05T11:15:38,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, REOPEN/MOVE because future has completed 2024-12-05T11:15:38,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:15:38,236 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,236 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:15:38,236 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,236 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:15:38,236 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,236 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,239 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,240 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:38,241 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72a0ecfe0b917c5e942c56777fb6984 columnFamilyName A 2024-12-05T11:15:38,243 DEBUG [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:38,243 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(327): Store=f72a0ecfe0b917c5e942c56777fb6984/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:38,244 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,245 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:38,245 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72a0ecfe0b917c5e942c56777fb6984 columnFamilyName B 2024-12-05T11:15:38,245 DEBUG [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:38,246 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(327): Store=f72a0ecfe0b917c5e942c56777fb6984/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:38,246 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,247 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:15:38,247 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72a0ecfe0b917c5e942c56777fb6984 columnFamilyName C 2024-12-05T11:15:38,247 DEBUG [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:38,248 INFO [StoreOpener-f72a0ecfe0b917c5e942c56777fb6984-1 {}] regionserver.HStore(327): Store=f72a0ecfe0b917c5e942c56777fb6984/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:15:38,248 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,248 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,248 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,249 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,250 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,250 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,251 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:15:38,252 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,252 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened f72a0ecfe0b917c5e942c56777fb6984; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67834808, jitterRate=0.010817408561706543}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:15:38,252 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,253 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for f72a0ecfe0b917c5e942c56777fb6984: Running coprocessor pre-open hook at 1733397338237Writing region info on filesystem at 1733397338237Initializing all the Stores at 1733397338238 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397338238Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397338238Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397338238Cleaning up temporary data from old regions at 1733397338250 (+12 ms)Running coprocessor post-open hooks at 1733397338252 (+2 ms)Region opened successfully at 1733397338253 (+1 ms) 2024-12-05T11:15:38,253 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., pid=84, masterSystemTime=1733397338232 2024-12-05T11:15:38,255 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,255 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,255 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=OPEN, openSeqNum=5, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:15:38,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-12-05T11:15:38,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 in 178 msec 2024-12-05T11:15:38,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-05T11:15:38,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, REOPEN/MOVE in 509 msec 2024-12-05T11:15:38,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-05T11:15:38,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-12-05T11:15:38,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-12-05T11:15:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T11:15:38,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:38,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac1e8e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,268 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6acec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,270 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fae4c78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781d850f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,271 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fae8b12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,273 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c0ce59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3da67742, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,275 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60077fab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,276 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ab0ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c40625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,277 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dee68dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,278 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27dc0022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4ab058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,280 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54778556, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,281 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14cf731, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2846e423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fc4fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7878be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1153f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,285 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@360b1449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,286 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b8db5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623dae41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,288 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5208f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3a684f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37755a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243519f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cb10b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520304d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:15:38,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:15:38,294 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:15:38,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f824d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:15:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:38,296 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:15:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc259d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:15:38,297 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,297 DEBUG [Thread-1431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,297 DEBUG [Thread-1438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,297 DEBUG [Thread-1434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,297 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,298 DEBUG [Thread-1442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,298 DEBUG [Thread-1429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-12-05T11:15:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,302 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:38,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,302 DEBUG [Thread-1440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,304 DEBUG [Thread-1427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,304 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-05T11:15:38,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,308 DEBUG [Thread-1436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:15:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:15:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:15:38,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:15:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:15:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:15:38,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:38,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:38,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:38,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:38,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:38,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397398334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fe070065fe4547dcb50d7925de4ef73a_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_1/A:col10/1733397338314/Put/seqid=0 2024-12-05T11:15:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397398335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397398336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397398336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397398339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742134_1310 (size=14594) 2024-12-05T11:15:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-05T11:15:38,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397398448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397398448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397398448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397398448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397398449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:38,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:38,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:38,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:38,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-05T11:15:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397398658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397398658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397398658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397398658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397398659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:38,754 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fe070065fe4547dcb50d7925de4ef73a_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fe070065fe4547dcb50d7925de4ef73a_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:38,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/e01364bbb2834f92b26bbf3ab1df234e, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:38,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/e01364bbb2834f92b26bbf3ab1df234e is 175, key is test_row_1/A:col10/1733397338314/Put/seqid=0 2024-12-05T11:15:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742135_1311 (size=39545) 2024-12-05T11:15:38,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:38,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-05T11:15:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397398968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397398968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397398968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397398968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397398969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:39,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:39,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/e01364bbb2834f92b26bbf3ab1df234e 2024-12-05T11:15:39,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/54a2b50c52b2467ab1c590c2874b53c7 is 50, key is test_row_1/B:col10/1733397338314/Put/seqid=0 2024-12-05T11:15:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742136_1312 (size=9657) 2024-12-05T11:15:39,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/54a2b50c52b2467ab1c590c2874b53c7 2024-12-05T11:15:39,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/e3c820d4992f4800ae7af76e4145d152 is 50, key is test_row_1/C:col10/1733397338314/Put/seqid=0 2024-12-05T11:15:39,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742137_1313 (size=9657) 2024-12-05T11:15:39,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/e3c820d4992f4800ae7af76e4145d152 2024-12-05T11:15:39,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:39,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:39,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:39,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/e01364bbb2834f92b26bbf3ab1df234e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e 2024-12-05T11:15:39,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e, entries=200, sequenceid=17, filesize=38.6 K 2024-12-05T11:15:39,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/54a2b50c52b2467ab1c590c2874b53c7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/54a2b50c52b2467ab1c590c2874b53c7 2024-12-05T11:15:39,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/54a2b50c52b2467ab1c590c2874b53c7, entries=100, sequenceid=17, filesize=9.4 K 2024-12-05T11:15:39,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/e3c820d4992f4800ae7af76e4145d152 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/e3c820d4992f4800ae7af76e4145d152 2024-12-05T11:15:39,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/e3c820d4992f4800ae7af76e4145d152, entries=100, sequenceid=17, filesize=9.4 K 2024-12-05T11:15:39,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f72a0ecfe0b917c5e942c56777fb6984 in 938ms, sequenceid=17, compaction requested=false 2024-12-05T11:15:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:39,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-05T11:15:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:39,377 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:15:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:39,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120599a49aabc9034b21ad7fbbfec69a78e5_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397338335/Put/seqid=0 2024-12-05T11:15:39,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742138_1314 (size=12154) 2024-12-05T11:15:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-05T11:15:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:39,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:39,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397399489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397399490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397399490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397399492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397399493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397399598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397399598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397399598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397399608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397399608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:39,793 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120599a49aabc9034b21ad7fbbfec69a78e5_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120599a49aabc9034b21ad7fbbfec69a78e5_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:39,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/d1abc4c7874d48328f005fe28135149c, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:39,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/d1abc4c7874d48328f005fe28135149c is 175, key is test_row_0/A:col10/1733397338335/Put/seqid=0 2024-12-05T11:15:39,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742139_1315 (size=30955) 2024-12-05T11:15:39,802 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/d1abc4c7874d48328f005fe28135149c 2024-12-05T11:15:39,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/831a50b9a3e748f1a4c272508d1971e4 is 50, key is test_row_0/B:col10/1733397338335/Put/seqid=0 2024-12-05T11:15:39,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397399808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397399808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397399809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742140_1316 (size=12001) 2024-12-05T11:15:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397399818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397399818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:39,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:15:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397400118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397400118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397400129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397400138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397400138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,225 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/831a50b9a3e748f1a4c272508d1971e4 2024-12-05T11:15:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/c2d549ec396b488e9c7d9963d68d594d is 50, key is test_row_0/C:col10/1733397338335/Put/seqid=0 2024-12-05T11:15:40,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742141_1317 (size=12001) 2024-12-05T11:15:40,245 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/c2d549ec396b488e9c7d9963d68d594d 2024-12-05T11:15:40,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/d1abc4c7874d48328f005fe28135149c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c 2024-12-05T11:15:40,259 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c, entries=150, sequenceid=36, filesize=30.2 K 2024-12-05T11:15:40,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/831a50b9a3e748f1a4c272508d1971e4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/831a50b9a3e748f1a4c272508d1971e4 2024-12-05T11:15:40,264 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/831a50b9a3e748f1a4c272508d1971e4, entries=150, sequenceid=36, filesize=11.7 K 2024-12-05T11:15:40,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/c2d549ec396b488e9c7d9963d68d594d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/c2d549ec396b488e9c7d9963d68d594d 2024-12-05T11:15:40,268 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/c2d549ec396b488e9c7d9963d68d594d, entries=150, sequenceid=36, filesize=11.7 K 2024-12-05T11:15:40,269 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f72a0ecfe0b917c5e942c56777fb6984 in 892ms, sequenceid=36, compaction requested=false 2024-12-05T11:15:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-05T11:15:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-05T11:15:40,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-05T11:15:40,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-12-05T11:15:40,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 1.9760 sec 2024-12-05T11:15:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-05T11:15:40,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:40,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-12-05T11:15:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:15:40,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:40,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:40,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:15:40,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-05T11:15:40,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:40,604 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:40,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051cd87feb1aa340539c0ebebbb763fc59_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397339491/Put/seqid=0 2024-12-05T11:15:40,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742142_1318 (size=12154) 2024-12-05T11:15:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:40,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:40,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397400654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397400656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397400657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397400659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397400660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:15:40,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397400768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397400768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397400769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397400769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397400769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397400978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397400979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397400979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397400979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397400988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:41,031 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051cd87feb1aa340539c0ebebbb763fc59_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051cd87feb1aa340539c0ebebbb763fc59_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9c80654fd43b46b79f040171f1c10830, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9c80654fd43b46b79f040171f1c10830 is 175, key is test_row_0/A:col10/1733397339491/Put/seqid=0 2024-12-05T11:15:41,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742143_1319 (size=30955) 2024-12-05T11:15:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:15:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397401288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397401288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397401288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397401288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397401298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,437 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9c80654fd43b46b79f040171f1c10830 2024-12-05T11:15:41,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/402548382b4d4d7eb1a639d3e2337ff3 is 50, key is test_row_0/B:col10/1733397339491/Put/seqid=0 2024-12-05T11:15:41,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742144_1320 (size=12001) 2024-12-05T11:15:41,470 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/402548382b4d4d7eb1a639d3e2337ff3 2024-12-05T11:15:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/cc869189f8fc422d89af807f11491fe8 is 50, key is test_row_0/C:col10/1733397339491/Put/seqid=0 2024-12-05T11:15:41,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742145_1321 (size=12001) 2024-12-05T11:15:41,514 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/cc869189f8fc422d89af807f11491fe8 2024-12-05T11:15:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9c80654fd43b46b79f040171f1c10830 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830 2024-12-05T11:15:41,524 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830, entries=150, sequenceid=49, filesize=30.2 K 2024-12-05T11:15:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/402548382b4d4d7eb1a639d3e2337ff3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/402548382b4d4d7eb1a639d3e2337ff3 2024-12-05T11:15:41,530 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/402548382b4d4d7eb1a639d3e2337ff3, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T11:15:41,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/cc869189f8fc422d89af807f11491fe8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/cc869189f8fc422d89af807f11491fe8 2024-12-05T11:15:41,537 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/cc869189f8fc422d89af807f11491fe8, entries=150, sequenceid=49, filesize=11.7 K 2024-12-05T11:15:41,538 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f72a0ecfe0b917c5e942c56777fb6984 in 935ms, sequenceid=49, compaction requested=true 2024-12-05T11:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-05T11:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-05T11:15:41,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-05T11:15:41,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0880 sec 2024-12-05T11:15:41,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.0930 sec 2024-12-05T11:15:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-05T11:15:41,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:41,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-05T11:15:41,591 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T11:15:41,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:41,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T11:15:41,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T11:15:41,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:41,745 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:41,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a745612dbfce4ad0bd8884a22e223ce0_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397340659/Put/seqid=0 2024-12-05T11:15:41,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742146_1322 (size=12154) 2024-12-05T11:15:41,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:41,781 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a745612dbfce4ad0bd8884a22e223ce0_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a745612dbfce4ad0bd8884a22e223ce0_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3c991d7970ef46fd986c18c28bcce033, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3c991d7970ef46fd986c18c28bcce033 is 175, key is test_row_0/A:col10/1733397340659/Put/seqid=0 2024-12-05T11:15:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:41,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742147_1323 (size=30955) 2024-12-05T11:15:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397401839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397401841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397401841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T11:15:41,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397401948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397401958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397401958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397401958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397401958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397402158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397402169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397402169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397402169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397402169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,211 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3c991d7970ef46fd986c18c28bcce033 2024-12-05T11:15:42,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/1831a89b06124faf86880ba947db07ac is 50, key is test_row_0/B:col10/1733397340659/Put/seqid=0 2024-12-05T11:15:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T11:15:42,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742148_1324 (size=12001) 2024-12-05T11:15:42,225 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/1831a89b06124faf86880ba947db07ac 2024-12-05T11:15:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/becbc51dbac043c4bb898e39326322a1 is 50, key is test_row_0/C:col10/1733397340659/Put/seqid=0 2024-12-05T11:15:42,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742149_1325 (size=12001) 2024-12-05T11:15:42,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/becbc51dbac043c4bb898e39326322a1 2024-12-05T11:15:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3c991d7970ef46fd986c18c28bcce033 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033 2024-12-05T11:15:42,279 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033, entries=150, sequenceid=68, filesize=30.2 K 2024-12-05T11:15:42,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/1831a89b06124faf86880ba947db07ac as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/1831a89b06124faf86880ba947db07ac 2024-12-05T11:15:42,291 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/1831a89b06124faf86880ba947db07ac, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T11:15:42,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/becbc51dbac043c4bb898e39326322a1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/becbc51dbac043c4bb898e39326322a1 2024-12-05T11:15:42,296 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/becbc51dbac043c4bb898e39326322a1, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T11:15:42,297 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for f72a0ecfe0b917c5e942c56777fb6984 in 553ms, sequenceid=68, compaction requested=true 2024-12-05T11:15:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-05T11:15:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-05T11:15:42,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-05T11:15:42,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 706 msec 2024-12-05T11:15:42,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 712 msec 2024-12-05T11:15:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:42,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T11:15:42,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:42,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:42,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:42,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:42,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:42,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:42,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120502fd2924bb2f4cccbb1483c93374eeb5_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742150_1326 (size=14594) 2024-12-05T11:15:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397402507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397402508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397402508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397402509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397402510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397402618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397402628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397402628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397402628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397402628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T11:15:42,729 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:42,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-05T11:15:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:42,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:42,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:42,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:42,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397402832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397402839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397402839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397402839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397402839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:42,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:42,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:42,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:42,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120502fd2924bb2f4cccbb1483c93374eeb5_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120502fd2924bb2f4cccbb1483c93374eeb5_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:42,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/8c81cb346e8c4258969b1ac17bc4cc28, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:42,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/8c81cb346e8c4258969b1ac17bc4cc28 is 175, key is test_row_0/A:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:42,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742151_1327 (size=39549) 2024-12-05T11:15:43,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:43,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397403149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397403149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397403149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397403149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397403149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/8c81cb346e8c4258969b1ac17bc4cc28 2024-12-05T11:15:43,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/dac37781271c467c8489d9da1172189c is 50, key is test_row_0/B:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:43,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742152_1328 (size=12001) 2024-12-05T11:15:43,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:43,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397403658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397403668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397403668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397403668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397403668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:43,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/dac37781271c467c8489d9da1172189c 2024-12-05T11:15:43,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/9874b67940be4a34b680fab3469bdd7b is 50, key is test_row_0/C:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:43,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742153_1329 (size=12001) 2024-12-05T11:15:43,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:43,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:43,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:43,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:43,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:44,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:44,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:44,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/9874b67940be4a34b680fab3469bdd7b 2024-12-05T11:15:44,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/8c81cb346e8c4258969b1ac17bc4cc28 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28 2024-12-05T11:15:44,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28, entries=200, sequenceid=83, filesize=38.6 K 2024-12-05T11:15:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/dac37781271c467c8489d9da1172189c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/dac37781271c467c8489d9da1172189c 2024-12-05T11:15:44,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/dac37781271c467c8489d9da1172189c, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T11:15:44,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/9874b67940be4a34b680fab3469bdd7b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9874b67940be4a34b680fab3469bdd7b 2024-12-05T11:15:44,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9874b67940be4a34b680fab3469bdd7b, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T11:15:44,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f72a0ecfe0b917c5e942c56777fb6984 in 1683ms, sequenceid=83, compaction requested=true 2024-12-05T11:15:44,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:44,166 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T11:15:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:44,166 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T11:15:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:44,168 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 171959 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T11:15:44,168 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T11:15:44,168 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/B is initiating minor compaction (all files) 2024-12-05T11:15:44,168 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/A is initiating minor compaction (all files) 2024-12-05T11:15:44,168 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/A in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,168 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=167.9 K 2024-12-05T11:15:44,168 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,169 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28] 2024-12-05T11:15:44,169 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/B in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,169 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/54a2b50c52b2467ab1c590c2874b53c7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/831a50b9a3e748f1a4c272508d1971e4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/402548382b4d4d7eb1a639d3e2337ff3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/1831a89b06124faf86880ba947db07ac, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/dac37781271c467c8489d9da1172189c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=56.3 K 2024-12-05T11:15:44,169 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e01364bbb2834f92b26bbf3ab1df234e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733397338312 2024-12-05T11:15:44,169 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 54a2b50c52b2467ab1c590c2874b53c7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733397338314 2024-12-05T11:15:44,170 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 831a50b9a3e748f1a4c272508d1971e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733397338334 2024-12-05T11:15:44,171 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 402548382b4d4d7eb1a639d3e2337ff3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733397339487 2024-12-05T11:15:44,171 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1abc4c7874d48328f005fe28135149c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733397338334 2024-12-05T11:15:44,172 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1831a89b06124faf86880ba947db07ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733397340656 2024-12-05T11:15:44,172 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c80654fd43b46b79f040171f1c10830, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733397339487 2024-12-05T11:15:44,172 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting dac37781271c467c8489d9da1172189c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733397341838 2024-12-05T11:15:44,172 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c991d7970ef46fd986c18c28bcce033, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733397340656 2024-12-05T11:15:44,172 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c81cb346e8c4258969b1ac17bc4cc28, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733397341838 2024-12-05T11:15:44,194 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:44,196 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#B#compaction#279 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:44,196 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/d84aa5a6c4ff4b9d9007de68afea552d is 50, key is test_row_0/B:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:44,209 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412050e2dd6eda9094449aef3b7331e1bf3f1_f72a0ecfe0b917c5e942c56777fb6984 store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:44,210 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412050e2dd6eda9094449aef3b7331e1bf3f1_f72a0ecfe0b917c5e942c56777fb6984, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:44,210 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050e2dd6eda9094449aef3b7331e1bf3f1_f72a0ecfe0b917c5e942c56777fb6984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:44,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742154_1330 (size=12173) 2024-12-05T11:15:44,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742155_1331 (size=4469) 2024-12-05T11:15:44,225 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#A#compaction#280 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:44,225 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/a5726c82a42741c3adbf57a4a49bcb13 is 175, key is test_row_0/A:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:44,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742156_1332 (size=31127) 2024-12-05T11:15:44,241 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/a5726c82a42741c3adbf57a4a49bcb13 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a5726c82a42741c3adbf57a4a49bcb13 2024-12-05T11:15:44,245 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/A of f72a0ecfe0b917c5e942c56777fb6984 into a5726c82a42741c3adbf57a4a49bcb13(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:44,245 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:44,245 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/A, priority=11, startTime=1733397344165; duration=0sec 2024-12-05T11:15:44,245 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:44,245 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:A 2024-12-05T11:15:44,245 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T11:15:44,246 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T11:15:44,246 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/C is initiating minor compaction (all files) 2024-12-05T11:15:44,246 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/C in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,246 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/e3c820d4992f4800ae7af76e4145d152, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/c2d549ec396b488e9c7d9963d68d594d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/cc869189f8fc422d89af807f11491fe8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/becbc51dbac043c4bb898e39326322a1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9874b67940be4a34b680fab3469bdd7b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=56.3 K 2024-12-05T11:15:44,247 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c820d4992f4800ae7af76e4145d152, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733397338314 2024-12-05T11:15:44,247 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2d549ec396b488e9c7d9963d68d594d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733397338334 2024-12-05T11:15:44,247 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc869189f8fc422d89af807f11491fe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733397339487 2024-12-05T11:15:44,247 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting becbc51dbac043c4bb898e39326322a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733397340656 2024-12-05T11:15:44,248 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9874b67940be4a34b680fab3469bdd7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733397341838 2024-12-05T11:15:44,256 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:44,256 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/49424104a6de45738ea799461c89339b is 50, key is test_row_0/C:col10/1733397341840/Put/seqid=0 2024-12-05T11:15:44,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742157_1333 (size=12173) 2024-12-05T11:15:44,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:44,260 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:44,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120571d6df7eac4f4c7fa9bcc2a3d0777ef3_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397342508/Put/seqid=0 2024-12-05T11:15:44,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742158_1334 (size=12154) 2024-12-05T11:15:44,624 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/d84aa5a6c4ff4b9d9007de68afea552d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/d84aa5a6c4ff4b9d9007de68afea552d 2024-12-05T11:15:44,630 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/B of f72a0ecfe0b917c5e942c56777fb6984 into d84aa5a6c4ff4b9d9007de68afea552d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:44,630 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:44,630 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/B, priority=11, startTime=1733397344166; duration=0sec 2024-12-05T11:15:44,630 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:44,630 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:B 2024-12-05T11:15:44,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:44,681 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/49424104a6de45738ea799461c89339b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/49424104a6de45738ea799461c89339b 2024-12-05T11:15:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:44,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:44,685 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120571d6df7eac4f4c7fa9bcc2a3d0777ef3_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120571d6df7eac4f4c7fa9bcc2a3d0777ef3_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:44,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/5e423570cd7e4801bd424e886b30572a, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:44,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/5e423570cd7e4801bd424e886b30572a is 175, key is test_row_0/A:col10/1733397342508/Put/seqid=0 2024-12-05T11:15:44,693 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/C of f72a0ecfe0b917c5e942c56777fb6984 into 49424104a6de45738ea799461c89339b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:44,693 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:44,693 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/C, priority=11, startTime=1733397344166; duration=0sec 2024-12-05T11:15:44,693 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:44,693 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:C 2024-12-05T11:15:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742159_1335 (size=30955) 2024-12-05T11:15:44,728 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/5e423570cd7e4801bd424e886b30572a 2024-12-05T11:15:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397404717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397404721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397404723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397404724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397404725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/4f46dc966a664ff8aab471bd05008b47 is 50, key is test_row_0/B:col10/1733397342508/Put/seqid=0 2024-12-05T11:15:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742160_1336 (size=12001) 2024-12-05T11:15:44,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397404838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397404839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397404848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397404848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397404848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:45,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397405048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397405049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397405059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397405059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397405059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,183 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/4f46dc966a664ff8aab471bd05008b47 2024-12-05T11:15:45,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/1d9d11630a1c406f8743f3e46a775f66 is 50, key is test_row_0/C:col10/1733397342508/Put/seqid=0 2024-12-05T11:15:45,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742161_1337 (size=12001) 2024-12-05T11:15:45,270 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/1d9d11630a1c406f8743f3e46a775f66 2024-12-05T11:15:45,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/5e423570cd7e4801bd424e886b30572a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a 2024-12-05T11:15:45,285 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a, entries=150, sequenceid=101, filesize=30.2 K 2024-12-05T11:15:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/4f46dc966a664ff8aab471bd05008b47 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/4f46dc966a664ff8aab471bd05008b47 2024-12-05T11:15:45,292 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/4f46dc966a664ff8aab471bd05008b47, entries=150, sequenceid=101, filesize=11.7 K 2024-12-05T11:15:45,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/1d9d11630a1c406f8743f3e46a775f66 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/1d9d11630a1c406f8743f3e46a775f66 2024-12-05T11:15:45,296 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/1d9d11630a1c406f8743f3e46a775f66, entries=150, sequenceid=101, filesize=11.7 K 2024-12-05T11:15:45,297 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f72a0ecfe0b917c5e942c56777fb6984 in 1037ms, sequenceid=101, compaction requested=false 2024-12-05T11:15:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-05T11:15:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-05T11:15:45,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T11:15:45,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5670 sec 2024-12-05T11:15:45,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.5770 sec 2024-12-05T11:15:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:45,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T11:15:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:45,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ce5c20da05d4d9fae8d445daf634cab_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:45,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742162_1338 (size=14594) 2024-12-05T11:15:45,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:45,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397405395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397405399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ce5c20da05d4d9fae8d445daf634cab_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ce5c20da05d4d9fae8d445daf634cab_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:45,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397405408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9d631ce5c3884806bd7f0b9b97e457a6, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9d631ce5c3884806bd7f0b9b97e457a6 is 175, key is test_row_0/A:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397405410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397405410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742163_1339 (size=39549) 2024-12-05T11:15:45,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9d631ce5c3884806bd7f0b9b97e457a6 2024-12-05T11:15:45,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/76e9650edea0445f8762fba5e0c32566 is 50, key is test_row_0/B:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397405520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397405521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397405529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397405538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397405538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742164_1340 (size=12001) 2024-12-05T11:15:45,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/76e9650edea0445f8762fba5e0c32566 2024-12-05T11:15:45,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/048fcf800c1349a88a9bf750afa350a2 is 50, key is test_row_0/C:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:45,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742165_1341 (size=12001) 2024-12-05T11:15:45,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397405738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397405738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397405759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397405759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397405759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/048fcf800c1349a88a9bf750afa350a2 2024-12-05T11:15:46,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9d631ce5c3884806bd7f0b9b97e457a6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6 2024-12-05T11:15:46,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6, entries=200, sequenceid=119, filesize=38.6 K 2024-12-05T11:15:46,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/76e9650edea0445f8762fba5e0c32566 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/76e9650edea0445f8762fba5e0c32566 2024-12-05T11:15:46,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/76e9650edea0445f8762fba5e0c32566, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T11:15:46,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/048fcf800c1349a88a9bf750afa350a2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/048fcf800c1349a88a9bf750afa350a2 2024-12-05T11:15:46,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/048fcf800c1349a88a9bf750afa350a2, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T11:15:46,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for f72a0ecfe0b917c5e942c56777fb6984 in 677ms, sequenceid=119, compaction requested=true 2024-12-05T11:15:46,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:46,041 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:46,041 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:46,041 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:46,041 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:46,041 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/A is initiating minor compaction (all files) 2024-12-05T11:15:46,041 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/B is initiating minor compaction (all files) 2024-12-05T11:15:46,041 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/A in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:46,041 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/B in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:46,042 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a5726c82a42741c3adbf57a4a49bcb13, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=99.2 K 2024-12-05T11:15:46,042 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/d84aa5a6c4ff4b9d9007de68afea552d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/4f46dc966a664ff8aab471bd05008b47, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/76e9650edea0445f8762fba5e0c32566] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=35.3 K 2024-12-05T11:15:46,042 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a5726c82a42741c3adbf57a4a49bcb13, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6] 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5726c82a42741c3adbf57a4a49bcb13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733397341838 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d84aa5a6c4ff4b9d9007de68afea552d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733397341838 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e423570cd7e4801bd424e886b30572a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733397342507 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f46dc966a664ff8aab471bd05008b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733397342507 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e9650edea0445f8762fba5e0c32566, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733397344717 2024-12-05T11:15:46,042 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d631ce5c3884806bd7f0b9b97e457a6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733397344717 2024-12-05T11:15:46,048 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:46,048 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/c28fe208d86f4f2f8216918ab84b76fd is 50, key is test_row_0/B:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:46,050 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:46,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742166_1342 (size=12275) 2024-12-05T11:15:46,055 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412051e8e0db688e848ecab90a7d19a91230c_f72a0ecfe0b917c5e942c56777fb6984 store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:46,057 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412051e8e0db688e848ecab90a7d19a91230c_f72a0ecfe0b917c5e942c56777fb6984, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:46,058 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051e8e0db688e848ecab90a7d19a91230c_f72a0ecfe0b917c5e942c56777fb6984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:46,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T11:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:46,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742167_1343 (size=4469) 2024-12-05T11:15:46,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055156011831594f2d98777c75f6037fe1_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397346059/Put/seqid=0 2024-12-05T11:15:46,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742168_1344 (size=14794) 2024-12-05T11:15:46,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:46,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055156011831594f2d98777c75f6037fe1_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055156011831594f2d98777c75f6037fe1_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:46,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/813bf0fbe5224e4ebe890e94af98d711, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:46,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/813bf0fbe5224e4ebe890e94af98d711 is 175, key is test_row_0/A:col10/1733397346059/Put/seqid=0 2024-12-05T11:15:46,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742169_1345 (size=39749) 2024-12-05T11:15:46,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397406079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397406087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397406088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397406089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397406090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397406198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397406409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397406428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397406428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397406428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397406428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,457 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/c28fe208d86f4f2f8216918ab84b76fd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c28fe208d86f4f2f8216918ab84b76fd 2024-12-05T11:15:46,461 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/B of f72a0ecfe0b917c5e942c56777fb6984 into c28fe208d86f4f2f8216918ab84b76fd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:46,461 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:46,461 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/B, priority=13, startTime=1733397346041; duration=0sec 2024-12-05T11:15:46,461 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:46,461 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:B 2024-12-05T11:15:46,461 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:46,462 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:46,462 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/C is initiating minor compaction (all files) 2024-12-05T11:15:46,462 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/C in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:46,463 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/49424104a6de45738ea799461c89339b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/1d9d11630a1c406f8743f3e46a775f66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/048fcf800c1349a88a9bf750afa350a2] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=35.3 K 2024-12-05T11:15:46,463 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 49424104a6de45738ea799461c89339b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733397341838 2024-12-05T11:15:46,463 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9d11630a1c406f8743f3e46a775f66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733397342507 2024-12-05T11:15:46,464 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 048fcf800c1349a88a9bf750afa350a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733397344717 2024-12-05T11:15:46,464 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#A#compaction#289 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:46,465 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/70f5c3e836c54ca181802ff0c6de2c15 is 175, key is test_row_0/A:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742170_1346 (size=31229) 2024-12-05T11:15:46,474 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:46,474 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/60a35f245340421e9e740e3bcdca3fb4 is 50, key is test_row_0/C:col10/1733397344720/Put/seqid=0 2024-12-05T11:15:46,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742171_1347 (size=12275) 2024-12-05T11:15:46,487 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/813bf0fbe5224e4ebe890e94af98d711 2024-12-05T11:15:46,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/f43bd0e499c04c5290560f692673d94e is 50, key is test_row_0/B:col10/1733397346059/Put/seqid=0 2024-12-05T11:15:46,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742172_1348 (size=12151) 2024-12-05T11:15:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397406718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397406738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397406738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397406738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397406738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:46,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:46,879 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/70f5c3e836c54ca181802ff0c6de2c15 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/70f5c3e836c54ca181802ff0c6de2c15 2024-12-05T11:15:46,882 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/60a35f245340421e9e740e3bcdca3fb4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/60a35f245340421e9e740e3bcdca3fb4 2024-12-05T11:15:46,883 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/A of f72a0ecfe0b917c5e942c56777fb6984 into 70f5c3e836c54ca181802ff0c6de2c15(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:46,883 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:46,883 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/A, priority=13, startTime=1733397346040; duration=0sec 2024-12-05T11:15:46,883 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:46,883 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:A 2024-12-05T11:15:46,886 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/C of f72a0ecfe0b917c5e942c56777fb6984 into 60a35f245340421e9e740e3bcdca3fb4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:46,886 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:46,886 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/C, priority=13, startTime=1733397346041; duration=0sec 2024-12-05T11:15:46,886 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:46,886 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:C 2024-12-05T11:15:46,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/f43bd0e499c04c5290560f692673d94e 2024-12-05T11:15:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T11:15:46,899 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:46,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-05T11:15:46,901 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T11:15:46,901 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:46,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:46,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/77a39fd4329c4d87b2f9d0337ff83f9d is 50, key is test_row_0/C:col10/1733397346059/Put/seqid=0 2024-12-05T11:15:46,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742173_1349 (size=12151) 2024-12-05T11:15:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T11:15:47,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T11:15:47,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:47,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:47,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:47,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T11:15:47,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:47,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:47,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:47,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:47,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T11:15:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397407239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:47,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397407248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397407250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:47,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397407259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:47,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397407259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:47,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/77a39fd4329c4d87b2f9d0337ff83f9d 2024-12-05T11:15:47,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/813bf0fbe5224e4ebe890e94af98d711 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711 2024-12-05T11:15:47,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711, entries=200, sequenceid=137, filesize=38.8 K 2024-12-05T11:15:47,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/f43bd0e499c04c5290560f692673d94e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f43bd0e499c04c5290560f692673d94e 2024-12-05T11:15:47,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f43bd0e499c04c5290560f692673d94e, entries=150, sequenceid=137, filesize=11.9 K 2024-12-05T11:15:47,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/77a39fd4329c4d87b2f9d0337ff83f9d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/77a39fd4329c4d87b2f9d0337ff83f9d 2024-12-05T11:15:47,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/77a39fd4329c4d87b2f9d0337ff83f9d, entries=150, sequenceid=137, filesize=11.9 K 2024-12-05T11:15:47,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f72a0ecfe0b917c5e942c56777fb6984 in 1277ms, sequenceid=137, compaction requested=false 2024-12-05T11:15:47,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:47,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:47,361 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:47,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a60ae375fd0a43b28272cab093aa6b4f_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397346086/Put/seqid=0 2024-12-05T11:15:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742174_1350 (size=12304) 2024-12-05T11:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T11:15:47,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:47,773 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a60ae375fd0a43b28272cab093aa6b4f_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a60ae375fd0a43b28272cab093aa6b4f_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:47,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/325f9d7b933c4227b25703847e6c6bfa, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:47,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/325f9d7b933c4227b25703847e6c6bfa is 175, key is test_row_0/A:col10/1733397346086/Put/seqid=0 2024-12-05T11:15:47,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742175_1351 (size=31105) 2024-12-05T11:15:47,778 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/325f9d7b933c4227b25703847e6c6bfa 2024-12-05T11:15:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/f7e041d599c64ff298cd7721c00b893f is 50, key is test_row_0/B:col10/1733397346086/Put/seqid=0 2024-12-05T11:15:47,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742176_1352 (size=12151) 2024-12-05T11:15:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T11:15:48,209 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/f7e041d599c64ff298cd7721c00b893f 2024-12-05T11:15:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/3178a0c06deb47fd85dd34284bc17cf1 is 50, key is test_row_0/C:col10/1733397346086/Put/seqid=0 2024-12-05T11:15:48,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742177_1353 (size=12151) 2024-12-05T11:15:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:48,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397408288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397408289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397408291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397408291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397408292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397408619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397408619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397408619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397408619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397408619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,634 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/3178a0c06deb47fd85dd34284bc17cf1 2024-12-05T11:15:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/325f9d7b933c4227b25703847e6c6bfa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa 2024-12-05T11:15:48,642 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa, entries=150, sequenceid=154, filesize=30.4 K 2024-12-05T11:15:48,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/f7e041d599c64ff298cd7721c00b893f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f7e041d599c64ff298cd7721c00b893f 2024-12-05T11:15:48,646 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f7e041d599c64ff298cd7721c00b893f, entries=150, sequenceid=154, filesize=11.9 K 2024-12-05T11:15:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/3178a0c06deb47fd85dd34284bc17cf1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/3178a0c06deb47fd85dd34284bc17cf1 2024-12-05T11:15:48,650 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/3178a0c06deb47fd85dd34284bc17cf1, entries=150, sequenceid=154, filesize=11.9 K 2024-12-05T11:15:48,651 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f72a0ecfe0b917c5e942c56777fb6984 in 1290ms, sequenceid=154, compaction requested=true 2024-12-05T11:15:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-05T11:15:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-05T11:15:48,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-05T11:15:48,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7500 sec 2024-12-05T11:15:48,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.7550 sec 2024-12-05T11:15:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:48,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:15:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:48,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1361893d2b64f8890665d443a7044f9_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:48,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742178_1354 (size=14794) 2024-12-05T11:15:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397408945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397408947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397408947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:48,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397408948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T11:15:49,049 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:49,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-05T11:15:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:15:49,051 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:49,051 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:49,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:49,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397409059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397409059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397409059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397409069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397409069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:15:49,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397409269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397409269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397409269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397409278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397409280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:49,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c1361893d2b64f8890665d443a7044f9_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1361893d2b64f8890665d443a7044f9_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:49,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/fa53a6cec65643e69f3184183de8d9c8, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:49,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/fa53a6cec65643e69f3184183de8d9c8 is 175, key is test_row_0/A:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:49,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742179_1355 (size=39749) 2024-12-05T11:15:49,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:15:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,357 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/fa53a6cec65643e69f3184183de8d9c8 2024-12-05T11:15:49,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/0b57c191961d44ba9e0b8ff7c13ac4df is 50, key is test_row_0/B:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:15:49,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742180_1356 (size=12151) 2024-12-05T11:15:49,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:15:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397409589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397409589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397409589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397409589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397409589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:49,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:15:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:49,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:15:49,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/0b57c191961d44ba9e0b8ff7c13ac4df 2024-12-05T11:15:49,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/8ad657b4d93a4e4cab792bf95d93bcdb is 50, key is test_row_0/C:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:49,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742181_1357 (size=12151) 2024-12-05T11:15:49,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/8ad657b4d93a4e4cab792bf95d93bcdb 2024-12-05T11:15:49,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:15:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:49,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/fa53a6cec65643e69f3184183de8d9c8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8 2024-12-05T11:15:49,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8, entries=200, sequenceid=173, filesize=38.8 K 2024-12-05T11:15:49,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/0b57c191961d44ba9e0b8ff7c13ac4df as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/0b57c191961d44ba9e0b8ff7c13ac4df 2024-12-05T11:15:49,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/0b57c191961d44ba9e0b8ff7c13ac4df, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:15:49,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/8ad657b4d93a4e4cab792bf95d93bcdb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/8ad657b4d93a4e4cab792bf95d93bcdb 2024-12-05T11:15:49,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/8ad657b4d93a4e4cab792bf95d93bcdb, entries=150, sequenceid=173, filesize=11.9 K 2024-12-05T11:15:49,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f72a0ecfe0b917c5e942c56777fb6984 in 922ms, sequenceid=173, compaction requested=true 2024-12-05T11:15:49,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:49,854 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:49,854 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:49,863 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141832 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:49,864 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/A is initiating minor compaction (all files) 2024-12-05T11:15:49,864 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/A in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,864 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/70f5c3e836c54ca181802ff0c6de2c15, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=138.5 K 2024-12-05T11:15:49,864 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,864 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/70f5c3e836c54ca181802ff0c6de2c15, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8] 2024-12-05T11:15:49,865 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70f5c3e836c54ca181802ff0c6de2c15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733397344717 2024-12-05T11:15:49,865 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:49,865 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 813bf0fbe5224e4ebe890e94af98d711, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733397345393 2024-12-05T11:15:49,865 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/B is initiating minor compaction (all files) 2024-12-05T11:15:49,866 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/B in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,866 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c28fe208d86f4f2f8216918ab84b76fd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f43bd0e499c04c5290560f692673d94e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f7e041d599c64ff298cd7721c00b893f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/0b57c191961d44ba9e0b8ff7c13ac4df] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=47.6 K 2024-12-05T11:15:49,867 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325f9d7b933c4227b25703847e6c6bfa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733397346086 2024-12-05T11:15:49,867 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c28fe208d86f4f2f8216918ab84b76fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733397344717 2024-12-05T11:15:49,870 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f43bd0e499c04c5290560f692673d94e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733397345393 2024-12-05T11:15:49,870 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa53a6cec65643e69f3184183de8d9c8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397348274 2024-12-05T11:15:49,870 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e041d599c64ff298cd7721c00b893f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733397346086 2024-12-05T11:15:49,871 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b57c191961d44ba9e0b8ff7c13ac4df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397348274 2024-12-05T11:15:49,879 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:49,893 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120538e69e07741a4b70be3e7ae3eb74fc76_f72a0ecfe0b917c5e942c56777fb6984 store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:49,895 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120538e69e07741a4b70be3e7ae3eb74fc76_f72a0ecfe0b917c5e942c56777fb6984, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:49,896 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120538e69e07741a4b70be3e7ae3eb74fc76_f72a0ecfe0b917c5e942c56777fb6984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:49,910 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:49,911 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/12b996ce4be448a787fe54b12b7f0b5a is 50, key is test_row_0/B:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742182_1358 (size=4469) 2024-12-05T11:15:49,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742183_1359 (size=12561) 2024-12-05T11:15:49,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T11:15:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:49,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:15:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205056f5f40d02947d797c1898d0c12784b_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397348944/Put/seqid=0 2024-12-05T11:15:50,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742184_1360 (size=12304) 2024-12-05T11:15:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:50,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:50,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397410141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397410146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397410147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397410149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397410158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:15:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397410269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397410269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397410270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397410271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397410278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,359 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#A#compaction#300 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:50,360 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/2bc7a3aee0f945c1ad54e12131bcb4ad is 175, key is test_row_0/A:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:50,365 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/12b996ce4be448a787fe54b12b7f0b5a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/12b996ce4be448a787fe54b12b7f0b5a 2024-12-05T11:15:50,369 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/B of f72a0ecfe0b917c5e942c56777fb6984 into 12b996ce4be448a787fe54b12b7f0b5a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:50,369 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:50,369 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/B, priority=12, startTime=1733397349854; duration=0sec 2024-12-05T11:15:50,369 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:50,369 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:B 2024-12-05T11:15:50,369 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:15:50,370 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:15:50,370 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/C is initiating minor compaction (all files) 2024-12-05T11:15:50,371 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/C in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:50,371 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/60a35f245340421e9e740e3bcdca3fb4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/77a39fd4329c4d87b2f9d0337ff83f9d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/3178a0c06deb47fd85dd34284bc17cf1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/8ad657b4d93a4e4cab792bf95d93bcdb] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=47.6 K 2024-12-05T11:15:50,371 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a35f245340421e9e740e3bcdca3fb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733397344717 2024-12-05T11:15:50,371 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a39fd4329c4d87b2f9d0337ff83f9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733397345393 2024-12-05T11:15:50,371 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3178a0c06deb47fd85dd34284bc17cf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733397346086 2024-12-05T11:15:50,372 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad657b4d93a4e4cab792bf95d93bcdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397348274 2024-12-05T11:15:50,382 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:50,382 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/7f4d3ec5ec22432385108c7ae45f08fa is 50, key is test_row_0/C:col10/1733397348274/Put/seqid=0 2024-12-05T11:15:50,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742185_1361 (size=31515) 2024-12-05T11:15:50,394 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/2bc7a3aee0f945c1ad54e12131bcb4ad as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/2bc7a3aee0f945c1ad54e12131bcb4ad 2024-12-05T11:15:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742186_1362 (size=12561) 2024-12-05T11:15:50,399 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/A of f72a0ecfe0b917c5e942c56777fb6984 into 2bc7a3aee0f945c1ad54e12131bcb4ad(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:50,400 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:50,400 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/A, priority=12, startTime=1733397349853; duration=0sec 2024-12-05T11:15:50,400 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:50,400 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:A 2024-12-05T11:15:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205056f5f40d02947d797c1898d0c12784b_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205056f5f40d02947d797c1898d0c12784b_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9a409d82881640c28ad0f5b9621ccdd9, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9a409d82881640c28ad0f5b9621ccdd9 is 175, key is test_row_0/A:col10/1733397348944/Put/seqid=0 2024-12-05T11:15:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742187_1363 (size=31105) 2024-12-05T11:15:50,442 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9a409d82881640c28ad0f5b9621ccdd9 2024-12-05T11:15:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/b33c981fce854a80b81d0bd879b13e7d is 50, key is test_row_0/B:col10/1733397348944/Put/seqid=0 2024-12-05T11:15:50,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742188_1364 (size=12151) 2024-12-05T11:15:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397410488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397410488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397410488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397410490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397410498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,800 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/7f4d3ec5ec22432385108c7ae45f08fa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/7f4d3ec5ec22432385108c7ae45f08fa 2024-12-05T11:15:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397410799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397410799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397410799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397410799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,806 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/C of f72a0ecfe0b917c5e942c56777fb6984 into 7f4d3ec5ec22432385108c7ae45f08fa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:50,806 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:50,806 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/C, priority=12, startTime=1733397349854; duration=0sec 2024-12-05T11:15:50,806 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:50,806 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:C 2024-12-05T11:15:50,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397410808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:50,858 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/b33c981fce854a80b81d0bd879b13e7d 2024-12-05T11:15:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/0d783ce99b364470b33d98802cc2abc2 is 50, key is test_row_0/C:col10/1733397348944/Put/seqid=0 2024-12-05T11:15:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742189_1365 (size=12151) 2024-12-05T11:15:50,869 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/0d783ce99b364470b33d98802cc2abc2 2024-12-05T11:15:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/9a409d82881640c28ad0f5b9621ccdd9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9 2024-12-05T11:15:50,878 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9, entries=150, sequenceid=186, filesize=30.4 K 2024-12-05T11:15:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/b33c981fce854a80b81d0bd879b13e7d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/b33c981fce854a80b81d0bd879b13e7d 2024-12-05T11:15:50,884 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/b33c981fce854a80b81d0bd879b13e7d, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T11:15:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/0d783ce99b364470b33d98802cc2abc2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/0d783ce99b364470b33d98802cc2abc2 2024-12-05T11:15:50,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/0d783ce99b364470b33d98802cc2abc2, entries=150, sequenceid=186, filesize=11.9 K 2024-12-05T11:15:50,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f72a0ecfe0b917c5e942c56777fb6984 in 910ms, sequenceid=186, compaction requested=false 2024-12-05T11:15:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-05T11:15:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-05T11:15:50,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T11:15:50,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8390 sec 2024-12-05T11:15:50,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8450 sec 2024-12-05T11:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T11:15:51,198 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:51,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-05T11:15:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:51,200 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:51,201 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:51,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:51,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:15:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:51,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056dcc16b994c64c779525a1e933c8e0a6_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:51,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742190_1366 (size=14794) 2024-12-05T11:15:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397411324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397411325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397411329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397411329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397411333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397411439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397411439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397411439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397411448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397411649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397411649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397411649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:51,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:51,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397411658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,725 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:51,728 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056dcc16b994c64c779525a1e933c8e0a6_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056dcc16b994c64c779525a1e933c8e0a6_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:51,729 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/b0c915c06b294ae6be1b14c33ebc787c, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:51,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/b0c915c06b294ae6be1b14c33ebc787c is 175, key is test_row_0/A:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:51,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742191_1367 (size=39749) 2024-12-05T11:15:51,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/b0c915c06b294ae6be1b14c33ebc787c 2024-12-05T11:15:51,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/90285905268840d593222525d3677557 is 50, key is test_row_0/B:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:51,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742192_1368 (size=12151) 2024-12-05T11:15:51,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:51,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:51,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:51,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397411969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397411969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397411969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397411978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:52,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/90285905268840d593222525d3677557 2024-12-05T11:15:52,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/52237e65360d427f812000cd617bd1f3 is 50, key is test_row_0/C:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:52,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742193_1369 (size=12151) 2024-12-05T11:15:52,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/52237e65360d427f812000cd617bd1f3 2024-12-05T11:15:52,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/b0c915c06b294ae6be1b14c33ebc787c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c 2024-12-05T11:15:52,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c, entries=200, sequenceid=209, filesize=38.8 K 2024-12-05T11:15:52,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/90285905268840d593222525d3677557 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/90285905268840d593222525d3677557 2024-12-05T11:15:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/90285905268840d593222525d3677557, entries=150, sequenceid=209, filesize=11.9 K 2024-12-05T11:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/52237e65360d427f812000cd617bd1f3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/52237e65360d427f812000cd617bd1f3 2024-12-05T11:15:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/52237e65360d427f812000cd617bd1f3, entries=150, sequenceid=209, filesize=11.9 K 2024-12-05T11:15:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f72a0ecfe0b917c5e942c56777fb6984 in 875ms, sequenceid=209, compaction requested=true 2024-12-05T11:15:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:52,186 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:52,187 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:52,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:52,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,188 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,188 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/A is initiating minor compaction (all files) 2024-12-05T11:15:52,188 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/A in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,188 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/2bc7a3aee0f945c1ad54e12131bcb4ad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=100.0 K 2024-12-05T11:15:52,188 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,188 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/2bc7a3aee0f945c1ad54e12131bcb4ad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c] 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,189 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:52,189 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/B is initiating minor compaction (all files) 2024-12-05T11:15:52,189 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/B in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,189 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/12b996ce4be448a787fe54b12b7f0b5a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/b33c981fce854a80b81d0bd879b13e7d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/90285905268840d593222525d3677557] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=36.0 K 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,189 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b996ce4be448a787fe54b12b7f0b5a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397348274 2024-12-05T11:15:52,189 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc7a3aee0f945c1ad54e12131bcb4ad, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397348274 2024-12-05T11:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,190 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b33c981fce854a80b81d0bd879b13e7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733397348944 2024-12-05T11:15:52,190 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a409d82881640c28ad0f5b9621ccdd9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733397348944 2024-12-05T11:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,190 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 90285905268840d593222525d3677557, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733397350140 2024-12-05T11:15:52,190 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c915c06b294ae6be1b14c33ebc787c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733397350140 2024-12-05T11:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,194 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,196 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412057eb5dc24b191451598d4016c5aeadcb7_f72a0ecfe0b917c5e942c56777fb6984 store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,198 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,198 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/9ba7e0bdd1e24bff8188b341ece446c0 is 50, key is test_row_0/B:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:52,198 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412057eb5dc24b191451598d4016c5aeadcb7_f72a0ecfe0b917c5e942c56777fb6984, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,198 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057eb5dc24b191451598d4016c5aeadcb7_f72a0ecfe0b917c5e942c56777fb6984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742194_1370 (size=12663) 2024-12-05T11:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,219 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/9ba7e0bdd1e24bff8188b341ece446c0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/9ba7e0bdd1e24bff8188b341ece446c0 2024-12-05T11:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742195_1371 (size=4469) 2024-12-05T11:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,224 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#A#compaction#309 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,224 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/5fd96820865d4693828c1787b9600eaa is 175, key is test_row_0/A:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,227 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/B of f72a0ecfe0b917c5e942c56777fb6984 into 9ba7e0bdd1e24bff8188b341ece446c0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:52,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:52,227 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/B, priority=13, startTime=1733397352186; duration=0sec 2024-12-05T11:15:52,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:52,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:B 2024-12-05T11:15:52,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,228 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:52,228 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/C is initiating minor compaction (all files) 2024-12-05T11:15:52,228 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/C in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,228 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/7f4d3ec5ec22432385108c7ae45f08fa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/0d783ce99b364470b33d98802cc2abc2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/52237e65360d427f812000cd617bd1f3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=36.0 K 2024-12-05T11:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,229 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f4d3ec5ec22432385108c7ae45f08fa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733397348274 2024-12-05T11:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,229 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d783ce99b364470b33d98802cc2abc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733397348944 2024-12-05T11:15:52,229 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 52237e65360d427f812000cd617bd1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733397350140 2024-12-05T11:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742196_1372 (size=31617) 2024-12-05T11:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,236 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/5fd96820865d4693828c1787b9600eaa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5fd96820865d4693828c1787b9600eaa 2024-12-05T11:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,241 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,242 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/5c5ef8642cc64bccbea635763ce157a4 is 50, key is test_row_0/C:col10/1733397350140/Put/seqid=0 2024-12-05T11:15:52,242 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/A of f72a0ecfe0b917c5e942c56777fb6984 into 5fd96820865d4693828c1787b9600eaa(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:52,242 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:52,242 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/A, priority=13, startTime=1733397352186; duration=0sec 2024-12-05T11:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,242 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:52,242 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:A 2024-12-05T11:15:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742197_1373 (size=12663) 2024-12-05T11:15:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,251 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/5c5ef8642cc64bccbea635763ce157a4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/5c5ef8642cc64bccbea635763ce157a4 2024-12-05T11:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,256 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/C of f72a0ecfe0b917c5e942c56777fb6984 into 5c5ef8642cc64bccbea635763ce157a4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:52,256 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:52,256 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/C, priority=13, startTime=1733397352187; duration=0sec 2024-12-05T11:15:52,257 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:52,257 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:C 2024-12-05T11:15:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T11:15:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:52,271 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T11:15:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:52,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cf3e11a38682439bb7147f36b7826560_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397351327/Put/seqid=0 2024-12-05T11:15:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742198_1374 (size=9814) 2024-12-05T11:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:52,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397412518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397412519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397412518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397412519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397412521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397412638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397412638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397412638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397412638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397412639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:52,687 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cf3e11a38682439bb7147f36b7826560_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cf3e11a38682439bb7147f36b7826560_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/7f14e866ef394e95908a53fd6f6d90be, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/7f14e866ef394e95908a53fd6f6d90be is 175, key is test_row_0/A:col10/1733397351327/Put/seqid=0 2024-12-05T11:15:52,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742199_1375 (size=22461) 2024-12-05T11:15:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397412848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397412848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397412848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397412848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397412849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,093 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/7f14e866ef394e95908a53fd6f6d90be 2024-12-05T11:15:53,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/34211c3358ba46038ec1cb293687ab6e is 50, key is test_row_0/B:col10/1733397351327/Put/seqid=0 2024-12-05T11:15:53,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742200_1376 (size=9757) 2024-12-05T11:15:53,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397413158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397413158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397413158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397413159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397413159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:53,502 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/34211c3358ba46038ec1cb293687ab6e 2024-12-05T11:15:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/a80c09f72d26494bbfb8d639cbb73d5c is 50, key is test_row_0/C:col10/1733397351327/Put/seqid=0 2024-12-05T11:15:53,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742201_1377 (size=9757) 2024-12-05T11:15:53,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397413668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397413669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397413669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397413669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397413669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:53,912 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/a80c09f72d26494bbfb8d639cbb73d5c 2024-12-05T11:15:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/7f14e866ef394e95908a53fd6f6d90be as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be 2024-12-05T11:15:53,919 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be, entries=100, sequenceid=224, filesize=21.9 K 2024-12-05T11:15:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/34211c3358ba46038ec1cb293687ab6e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/34211c3358ba46038ec1cb293687ab6e 2024-12-05T11:15:53,923 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/34211c3358ba46038ec1cb293687ab6e, entries=100, sequenceid=224, filesize=9.5 K 2024-12-05T11:15:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/a80c09f72d26494bbfb8d639cbb73d5c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/a80c09f72d26494bbfb8d639cbb73d5c 2024-12-05T11:15:53,928 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/a80c09f72d26494bbfb8d639cbb73d5c, entries=100, sequenceid=224, filesize=9.5 K 2024-12-05T11:15:53,929 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f72a0ecfe0b917c5e942c56777fb6984 in 1658ms, sequenceid=224, compaction requested=false 2024-12-05T11:15:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:53,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-05T11:15:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-05T11:15:53,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-05T11:15:53,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7290 sec 2024-12-05T11:15:53,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7330 sec 2024-12-05T11:15:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:54,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T11:15:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:54,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205aeb1a165bf904af2bf098a00424d6336_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742202_1378 (size=14794) 2024-12-05T11:15:54,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397414697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397414697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397414698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397414699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397414700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397414809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397414809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397414818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397414818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397414818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397415019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397415019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397415028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397415028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397415028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:55,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205aeb1a165bf904af2bf098a00424d6336_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205aeb1a165bf904af2bf098a00424d6336_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:55,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3172ed152a38419690ad4b7149a4faae, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:55,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3172ed152a38419690ad4b7149a4faae is 175, key is test_row_0/A:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742203_1379 (size=39749) 2024-12-05T11:15:55,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397415329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397415330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397415338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397415338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397415339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T11:15:55,359 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:55,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-05T11:15:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:15:55,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:55,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:55,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:15:55,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3172ed152a38419690ad4b7149a4faae 2024-12-05T11:15:55,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/544ea01f5a8f45a99dfa391efe16cd17 is 50, key is test_row_0/B:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:55,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:15:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742204_1380 (size=12151) 2024-12-05T11:15:55,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:15:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:15:55,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:15:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397415849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397415849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397415849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397415849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397415858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:55,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:55,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/544ea01f5a8f45a99dfa391efe16cd17 2024-12-05T11:15:55,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/84f43960116e40c5b7de38ec0faaf490 is 50, key is test_row_0/C:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:55,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742205_1381 (size=12151) 2024-12-05T11:15:55,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/84f43960116e40c5b7de38ec0faaf490 2024-12-05T11:15:55,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/3172ed152a38419690ad4b7149a4faae as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae 2024-12-05T11:15:55,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae, entries=200, sequenceid=245, filesize=38.8 K 2024-12-05T11:15:55,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/544ea01f5a8f45a99dfa391efe16cd17 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/544ea01f5a8f45a99dfa391efe16cd17 2024-12-05T11:15:55,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/544ea01f5a8f45a99dfa391efe16cd17, entries=150, sequenceid=245, filesize=11.9 K 2024-12-05T11:15:55,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/84f43960116e40c5b7de38ec0faaf490 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/84f43960116e40c5b7de38ec0faaf490 2024-12-05T11:15:55,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:15:55,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:15:55,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/84f43960116e40c5b7de38ec0faaf490, entries=150, sequenceid=245, filesize=11.9 K 2024-12-05T11:15:55,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f72a0ecfe0b917c5e942c56777fb6984 in 1313ms, sequenceid=245, compaction requested=true 2024-12-05T11:15:55,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:55,996 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:55,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:55,997 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:55,997 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/A is initiating minor compaction (all files) 2024-12-05T11:15:55,997 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/A in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,997 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5fd96820865d4693828c1787b9600eaa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=91.6 K 2024-12-05T11:15:55,997 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:55,997 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5fd96820865d4693828c1787b9600eaa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae] 2024-12-05T11:15:55,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:55,999 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd96820865d4693828c1787b9600eaa, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733397350140 2024-12-05T11:15:55,999 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:55,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:55,999 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f14e866ef394e95908a53fd6f6d90be, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733397351327 2024-12-05T11:15:56,000 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3172ed152a38419690ad4b7149a4faae, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397352499 2024-12-05T11:15:56,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:56,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:56,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:56,000 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:56,000 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/B is initiating minor compaction (all files) 2024-12-05T11:15:56,000 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/B in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:56,000 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/9ba7e0bdd1e24bff8188b341ece446c0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/34211c3358ba46038ec1cb293687ab6e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/544ea01f5a8f45a99dfa391efe16cd17] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=33.8 K 2024-12-05T11:15:56,001 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba7e0bdd1e24bff8188b341ece446c0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733397350140 2024-12-05T11:15:56,001 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 34211c3358ba46038ec1cb293687ab6e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733397351327 2024-12-05T11:15:56,001 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 544ea01f5a8f45a99dfa391efe16cd17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397352499 2024-12-05T11:15:56,007 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:56,026 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:56,027 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/59576d5ecfb64896bbb5d5f5a9d18cec is 50, key is test_row_0/B:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:56,033 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205a7735b17c2874c8581ed41a342e0452a_f72a0ecfe0b917c5e942c56777fb6984 store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:56,034 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205a7735b17c2874c8581ed41a342e0452a_f72a0ecfe0b917c5e942c56777fb6984, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:56,035 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a7735b17c2874c8581ed41a342e0452a_f72a0ecfe0b917c5e942c56777fb6984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:56,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742206_1382 (size=12765) 2024-12-05T11:15:56,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742207_1383 (size=4469) 2024-12-05T11:15:56,098 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/59576d5ecfb64896bbb5d5f5a9d18cec as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/59576d5ecfb64896bbb5d5f5a9d18cec 2024-12-05T11:15:56,099 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#A#compaction#318 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:56,099 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/a780e9949cbe432298e52c03a55ad798 is 175, key is test_row_0/A:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:56,114 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/B of f72a0ecfe0b917c5e942c56777fb6984 into 59576d5ecfb64896bbb5d5f5a9d18cec(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:56,114 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:56,114 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/B, priority=13, startTime=1733397355999; duration=0sec 2024-12-05T11:15:56,114 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:56,114 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:B 2024-12-05T11:15:56,115 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:56,116 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:56,116 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/C is initiating minor compaction (all files) 2024-12-05T11:15:56,116 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/C in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:56,117 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/5c5ef8642cc64bccbea635763ce157a4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/a80c09f72d26494bbfb8d639cbb73d5c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/84f43960116e40c5b7de38ec0faaf490] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=33.8 K 2024-12-05T11:15:56,117 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c5ef8642cc64bccbea635763ce157a4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733397350140 2024-12-05T11:15:56,118 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting a80c09f72d26494bbfb8d639cbb73d5c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733397351327 2024-12-05T11:15:56,118 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f43960116e40c5b7de38ec0faaf490, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397352499 2024-12-05T11:15:56,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742208_1384 (size=31719) 2024-12-05T11:15:56,135 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:56,136 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/2d2a61c8a1274e5894b8903b9f664b73 is 50, key is test_row_0/C:col10/1733397352517/Put/seqid=0 2024-12-05T11:15:56,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:56,139 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:56,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742209_1385 (size=12765) 2024-12-05T11:15:56,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ea1aa7e6064e4d949cb1d24b8ed9e69b_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397354698/Put/seqid=0 2024-12-05T11:15:56,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742210_1386 (size=12304) 2024-12-05T11:15:56,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,159 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ea1aa7e6064e4d949cb1d24b8ed9e69b_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ea1aa7e6064e4d949cb1d24b8ed9e69b_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/f75e50ff3c994ff1b885f5338c02d69e, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/f75e50ff3c994ff1b885f5338c02d69e is 175, key is test_row_0/A:col10/1733397354698/Put/seqid=0 2024-12-05T11:15:56,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742211_1387 (size=31105) 2024-12-05T11:15:56,169 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/f75e50ff3c994ff1b885f5338c02d69e 2024-12-05T11:15:56,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/e4f5dab24869435980d325f81813d2bb is 50, key is test_row_0/B:col10/1733397354698/Put/seqid=0 2024-12-05T11:15:56,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742212_1388 (size=12151) 2024-12-05T11:15:56,201 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/e4f5dab24869435980d325f81813d2bb 2024-12-05T11:15:56,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/51e5c21d769049b3a7d6b35d152e19ea is 50, key is test_row_0/C:col10/1733397354698/Put/seqid=0 2024-12-05T11:15:56,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742213_1389 (size=12151) 2024-12-05T11:15:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:15:56,534 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/a780e9949cbe432298e52c03a55ad798 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a780e9949cbe432298e52c03a55ad798 2024-12-05T11:15:56,540 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/A of f72a0ecfe0b917c5e942c56777fb6984 into a780e9949cbe432298e52c03a55ad798(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:56,540 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:56,540 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/A, priority=13, startTime=1733397355996; duration=0sec 2024-12-05T11:15:56,540 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:56,540 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:A 2024-12-05T11:15:56,553 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/2d2a61c8a1274e5894b8903b9f664b73 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/2d2a61c8a1274e5894b8903b9f664b73 2024-12-05T11:15:56,557 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/C of f72a0ecfe0b917c5e942c56777fb6984 into 2d2a61c8a1274e5894b8903b9f664b73(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:56,557 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:56,557 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/C, priority=13, startTime=1733397356000; duration=0sec 2024-12-05T11:15:56,557 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:56,557 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:C 2024-12-05T11:15:56,614 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/51e5c21d769049b3a7d6b35d152e19ea 2024-12-05T11:15:56,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/f75e50ff3c994ff1b885f5338c02d69e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e 2024-12-05T11:15:56,621 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e, entries=150, sequenceid=257, filesize=30.4 K 2024-12-05T11:15:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/e4f5dab24869435980d325f81813d2bb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/e4f5dab24869435980d325f81813d2bb 2024-12-05T11:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,625 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/e4f5dab24869435980d325f81813d2bb, entries=150, sequenceid=257, filesize=11.9 K 2024-12-05T11:15:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/51e5c21d769049b3a7d6b35d152e19ea as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/51e5c21d769049b3a7d6b35d152e19ea 2024-12-05T11:15:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,629 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/51e5c21d769049b3a7d6b35d152e19ea, entries=150, sequenceid=257, filesize=11.9 K 2024-12-05T11:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,630 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for f72a0ecfe0b917c5e942c56777fb6984 in 491ms, sequenceid=257, compaction requested=false 2024-12-05T11:15:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T11:15:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-05T11:15:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T11:15:56,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2690 sec 2024-12-05T11:15:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.2760 sec 2024-12-05T11:15:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:15:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:15:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d7960bb8c3f24f8eb1f9e311407f174d_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742214_1390 (size=22618) 2024-12-05T11:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397357004, max timestamp=1733397357030Num uniques 859; 2024-12-05T11:15:57,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:57,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397357004, max timestamp=1733397357030Num uniques 859; 2024-12-05T11:15:57,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:57,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397357004, max timestamp=1733397357030Num uniques 859; 2024-12-05T11:15:57,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:15:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397417066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397417069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397417071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397417072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397417078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397417188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397417188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397417199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397417199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397417199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:57,390 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d7960bb8c3f24f8eb1f9e311407f174d_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d7960bb8c3f24f8eb1f9e311407f174d_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:57,391 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/63d5a49b7c5042edb8bfc0d85cc9e177, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:57,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/63d5a49b7c5042edb8bfc0d85cc9e177 is 175, key is test_row_0/A:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:57,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742215_1391 (size=66023) 2024-12-05T11:15:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397417398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397417398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397417419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397417419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397417419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T11:15:57,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:15:57,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-05T11:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:15:57,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:15:57,511 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:15:57,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:15:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:15:57,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:57,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397417708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397417709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397417728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397417728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397417728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:57,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/63d5a49b7c5042edb8bfc0d85cc9e177 2024-12-05T11:15:57,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/60b369ee97894a5687850e82085814a6 is 50, key is test_row_0/B:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:57,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742216_1392 (size=12301) 2024-12-05T11:15:57,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:57,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:15:57,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:57,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:15:58,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/60b369ee97894a5687850e82085814a6 2024-12-05T11:15:58,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/aed32ac323f9430796250bfe28cd7ba1 is 50, key is test_row_0/C:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:58,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742217_1393 (size=12301) 2024-12-05T11:15:58,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38144 deadline: 1733397418228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:58,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38184 deadline: 1733397418228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38206 deadline: 1733397418238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38132 deadline: 1733397418239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38168 deadline: 1733397418239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:15:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:15:58,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,308 INFO [Thread-1434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1434. 2024-12-05T11:15:58,308 DEBUG [Thread-1434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:58,308 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,308 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:58,310 INFO [Thread-1438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1438. 2024-12-05T11:15:58,310 DEBUG [Thread-1438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:58,310 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,310 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:58,312 INFO [Thread-1440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1440. 2024-12-05T11:15:58,312 DEBUG [Thread-1440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:58,312 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,312 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:58,317 INFO [Thread-1442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1442. 2024-12-05T11:15:58,317 DEBUG [Thread-1442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:58,317 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,317 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:58,318 INFO [Thread-1436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1436. 2024-12-05T11:15:58,318 DEBUG [Thread-1436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:58,318 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,318 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:58,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:58,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:58,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:15:58,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/aed32ac323f9430796250bfe28cd7ba1 2024-12-05T11:15:58,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/63d5a49b7c5042edb8bfc0d85cc9e177 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177 2024-12-05T11:15:58,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177, entries=350, sequenceid=269, filesize=64.5 K 2024-12-05T11:15:58,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/60b369ee97894a5687850e82085814a6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/60b369ee97894a5687850e82085814a6 2024-12-05T11:15:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/60b369ee97894a5687850e82085814a6, entries=150, sequenceid=269, filesize=12.0 K 2024-12-05T11:15:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/aed32ac323f9430796250bfe28cd7ba1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/aed32ac323f9430796250bfe28cd7ba1 2024-12-05T11:15:58,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/aed32ac323f9430796250bfe28cd7ba1, entries=150, sequenceid=269, filesize=12.0 K 2024-12-05T11:15:58,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f72a0ecfe0b917c5e942c56777fb6984 in 1664ms, sequenceid=269, compaction requested=true 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:58,633 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f72a0ecfe0b917c5e942c56777fb6984:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:15:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:58,633 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:58,634 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:58,634 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128847 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:58,634 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/B is initiating minor compaction (all files) 2024-12-05T11:15:58,634 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/A is initiating minor compaction (all files) 2024-12-05T11:15:58,634 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/B in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,634 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/A in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,634 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/59576d5ecfb64896bbb5d5f5a9d18cec, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/e4f5dab24869435980d325f81813d2bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/60b369ee97894a5687850e82085814a6] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=36.3 K 2024-12-05T11:15:58,634 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a780e9949cbe432298e52c03a55ad798, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=125.8 K 2024-12-05T11:15:58,634 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,634 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a780e9949cbe432298e52c03a55ad798, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177] 2024-12-05T11:15:58,635 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 59576d5ecfb64896bbb5d5f5a9d18cec, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397352499 2024-12-05T11:15:58,635 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting a780e9949cbe432298e52c03a55ad798, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397352499 2024-12-05T11:15:58,635 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f5dab24869435980d325f81813d2bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733397354697 2024-12-05T11:15:58,635 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f75e50ff3c994ff1b885f5338c02d69e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733397354697 2024-12-05T11:15:58,635 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b369ee97894a5687850e82085814a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733397356936 2024-12-05T11:15:58,635 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d5a49b7c5042edb8bfc0d85cc9e177, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733397356936 2024-12-05T11:15:58,641 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:58,643 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:58,643 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/c61325bab0df46e784a5e8a76b90748f is 50, key is test_row_0/B:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:58,643 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c04242556abb465e8d65631c704f6d37_f72a0ecfe0b917c5e942c56777fb6984 store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:15:58,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742218_1394 (size=13017) 2024-12-05T11:15:58,651 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c04242556abb465e8d65631c704f6d37_f72a0ecfe0b917c5e942c56777fb6984, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:58,651 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c04242556abb465e8d65631c704f6d37_f72a0ecfe0b917c5e942c56777fb6984 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:58,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742219_1395 (size=4469) 2024-12-05T11:15:58,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:58,742 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:15:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:15:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050296223233fa4bcc86a962eff32e520e_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397357075/Put/seqid=0 2024-12-05T11:15:58,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742220_1396 (size=12454) 2024-12-05T11:15:59,054 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/c61325bab0df46e784a5e8a76b90748f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c61325bab0df46e784a5e8a76b90748f 2024-12-05T11:15:59,056 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:59,057 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/89d06a633cb14f02a935cdb5c46ba43d is 175, key is test_row_0/A:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:59,059 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/B of f72a0ecfe0b917c5e942c56777fb6984 into c61325bab0df46e784a5e8a76b90748f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:59,059 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:59,059 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/B, priority=13, startTime=1733397358633; duration=0sec 2024-12-05T11:15:59,059 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:15:59,059 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:B 2024-12-05T11:15:59,059 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:15:59,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:15:59,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): f72a0ecfe0b917c5e942c56777fb6984/C is initiating minor compaction (all files) 2024-12-05T11:15:59,060 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f72a0ecfe0b917c5e942c56777fb6984/C in TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:15:59,060 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/2d2a61c8a1274e5894b8903b9f664b73, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/51e5c21d769049b3a7d6b35d152e19ea, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/aed32ac323f9430796250bfe28cd7ba1] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp, totalSize=36.3 K 2024-12-05T11:15:59,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d2a61c8a1274e5894b8903b9f664b73, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397352499 2024-12-05T11:15:59,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e5c21d769049b3a7d6b35d152e19ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733397354697 2024-12-05T11:15:59,060 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting aed32ac323f9430796250bfe28cd7ba1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733397356936 2024-12-05T11:15:59,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742221_1397 (size=31971) 2024-12-05T11:15:59,069 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f72a0ecfe0b917c5e942c56777fb6984#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:15:59,069 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/9eb2f17e662840d3ab612c3d9e35c664 is 50, key is test_row_0/C:col10/1733397356944/Put/seqid=0 2024-12-05T11:15:59,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742222_1398 (size=13017) 2024-12-05T11:15:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:15:59,162 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050296223233fa4bcc86a962eff32e520e_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050296223233fa4bcc86a962eff32e520e_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:59,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/ffaa2e30211f4799854a7a59381a89a8, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:15:59,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/ffaa2e30211f4799854a7a59381a89a8 is 175, key is test_row_0/A:col10/1733397357075/Put/seqid=0 2024-12-05T11:15:59,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742223_1399 (size=31255) 2024-12-05T11:15:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:15:59,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. as already flushing 2024-12-05T11:15:59,240 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-12-05T11:15:59,240 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:59,240 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,240 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:59,250 INFO [Thread-1429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1429. 2024-12-05T11:15:59,250 DEBUG [Thread-1429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:59,250 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,250 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:59,260 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-12-05T11:15:59,260 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:59,260 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,260 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:59,260 INFO [Thread-1427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1427. 2024-12-05T11:15:59,260 DEBUG [Thread-1427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:59,260 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,260 INFO [Thread-1431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1431. 2024-12-05T11:15:59,261 DEBUG [Thread-1431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:15:59,261 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,261 DEBUG [Thread-1431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:59,261 DEBUG [Thread-1431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:15:59,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:15:59,473 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/89d06a633cb14f02a935cdb5c46ba43d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/89d06a633cb14f02a935cdb5c46ba43d 2024-12-05T11:15:59,479 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/9eb2f17e662840d3ab612c3d9e35c664 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9eb2f17e662840d3ab612c3d9e35c664 2024-12-05T11:15:59,479 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/A of f72a0ecfe0b917c5e942c56777fb6984 into 89d06a633cb14f02a935cdb5c46ba43d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:59,479 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:59,479 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/A, priority=13, startTime=1733397358633; duration=0sec 2024-12-05T11:15:59,479 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:59,479 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:A 2024-12-05T11:15:59,485 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f72a0ecfe0b917c5e942c56777fb6984/C of f72a0ecfe0b917c5e942c56777fb6984 into 9eb2f17e662840d3ab612c3d9e35c664(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:15:59,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:15:59,486 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., storeName=f72a0ecfe0b917c5e942c56777fb6984/C, priority=13, startTime=1733397358633; duration=0sec 2024-12-05T11:15:59,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:15:59,486 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f72a0ecfe0b917c5e942c56777fb6984:C 2024-12-05T11:15:59,569 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/ffaa2e30211f4799854a7a59381a89a8 2024-12-05T11:15:59,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/6c91e5561fd54a9eb0bc9912d4005faf is 50, key is test_row_0/B:col10/1733397357075/Put/seqid=0 2024-12-05T11:15:59,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742224_1400 (size=12301) 2024-12-05T11:15:59,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:15:59,980 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/6c91e5561fd54a9eb0bc9912d4005faf 2024-12-05T11:16:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/56e0f0ad6ff0405d963466673e16c03d is 50, key is test_row_0/C:col10/1733397357075/Put/seqid=0 2024-12-05T11:16:00,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742225_1401 (size=12301) 2024-12-05T11:16:00,444 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/56e0f0ad6ff0405d963466673e16c03d 2024-12-05T11:16:00,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/ffaa2e30211f4799854a7a59381a89a8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/ffaa2e30211f4799854a7a59381a89a8 2024-12-05T11:16:00,455 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/ffaa2e30211f4799854a7a59381a89a8, entries=150, sequenceid=306, filesize=30.5 K 2024-12-05T11:16:00,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/6c91e5561fd54a9eb0bc9912d4005faf as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/6c91e5561fd54a9eb0bc9912d4005faf 2024-12-05T11:16:00,462 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/6c91e5561fd54a9eb0bc9912d4005faf, entries=150, sequenceid=306, filesize=12.0 K 2024-12-05T11:16:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/56e0f0ad6ff0405d963466673e16c03d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/56e0f0ad6ff0405d963466673e16c03d 2024-12-05T11:16:00,467 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/56e0f0ad6ff0405d963466673e16c03d, entries=150, sequenceid=306, filesize=12.0 K 2024-12-05T11:16:00,468 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=33.54 KB/34350 for f72a0ecfe0b917c5e942c56777fb6984 in 1726ms, sequenceid=306, compaction requested=false 2024-12-05T11:16:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for f72a0ecfe0b917c5e942c56777fb6984: 2024-12-05T11:16:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:16:00,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-05T11:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-05T11:16:00,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-05T11:16:00,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9580 sec 2024-12-05T11:16:00,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.9630 sec 2024-12-05T11:16:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T11:16:01,678 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1559 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4673 rows 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1559 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4677 rows 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1562 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4681 rows 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1561 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4681 rows 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1556 2024-12-05T11:16:01,679 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4665 rows 2024-12-05T11:16:01,679 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:16:01,679 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:16:01,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:01,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:01,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:01,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:16:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:16:01,684 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397361684"}]},"ts":"1733397361684"} 2024-12-05T11:16:01,686 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:16:01,686 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:16:01,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:16:01,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, UNASSIGN}] 2024-12-05T11:16:01,690 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, UNASSIGN 2024-12-05T11:16:01,691 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:01,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, UNASSIGN because future has completed 2024-12-05T11:16:01,693 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:16:01,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:16:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:16:01,845 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:01,845 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:16:01,845 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing f72a0ecfe0b917c5e942c56777fb6984, disabling compactions & flushes 2024-12-05T11:16:01,845 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:16:01,845 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:16:01,845 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. after waiting 0 ms 2024-12-05T11:16:01,845 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:16:01,845 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing f72a0ecfe0b917c5e942c56777fb6984 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T11:16:01,845 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=A 2024-12-05T11:16:01,846 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:01,846 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=B 2024-12-05T11:16:01,846 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:01,846 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f72a0ecfe0b917c5e942c56777fb6984, store=C 2024-12-05T11:16:01,846 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:01,850 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055910495191a1467db201641fd3893393_f72a0ecfe0b917c5e942c56777fb6984 is 50, key is test_row_0/A:col10/1733397359259/Put/seqid=0 2024-12-05T11:16:01,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742226_1402 (size=12454) 2024-12-05T11:16:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:16:02,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:16:02,254 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:02,257 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055910495191a1467db201641fd3893393_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055910495191a1467db201641fd3893393_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:02,257 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/c0e3679629ee464aa46b871bc5781c1f, store: [table=TestAcidGuarantees family=A region=f72a0ecfe0b917c5e942c56777fb6984] 2024-12-05T11:16:02,258 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/c0e3679629ee464aa46b871bc5781c1f is 175, key is test_row_0/A:col10/1733397359259/Put/seqid=0 2024-12-05T11:16:02,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742227_1403 (size=31255) 2024-12-05T11:16:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:16:02,662 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/c0e3679629ee464aa46b871bc5781c1f 2024-12-05T11:16:02,674 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/2f80d3288b764b54a31c267b831b4f86 is 50, key is test_row_0/B:col10/1733397359259/Put/seqid=0 2024-12-05T11:16:02,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742228_1404 (size=12301) 2024-12-05T11:16:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:16:03,083 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/2f80d3288b764b54a31c267b831b4f86 2024-12-05T11:16:03,097 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/54c6a8926bf24805a614a456549c930c is 50, key is test_row_0/C:col10/1733397359259/Put/seqid=0 2024-12-05T11:16:03,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742229_1405 (size=12301) 2024-12-05T11:16:03,503 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/54c6a8926bf24805a614a456549c930c 2024-12-05T11:16:03,510 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/A/c0e3679629ee464aa46b871bc5781c1f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/c0e3679629ee464aa46b871bc5781c1f 2024-12-05T11:16:03,516 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/c0e3679629ee464aa46b871bc5781c1f, entries=150, sequenceid=317, filesize=30.5 K 2024-12-05T11:16:03,517 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/B/2f80d3288b764b54a31c267b831b4f86 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/2f80d3288b764b54a31c267b831b4f86 2024-12-05T11:16:03,521 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/2f80d3288b764b54a31c267b831b4f86, entries=150, sequenceid=317, filesize=12.0 K 2024-12-05T11:16:03,521 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/.tmp/C/54c6a8926bf24805a614a456549c930c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/54c6a8926bf24805a614a456549c930c 2024-12-05T11:16:03,525 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/54c6a8926bf24805a614a456549c930c, entries=150, sequenceid=317, filesize=12.0 K 2024-12-05T11:16:03,525 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f72a0ecfe0b917c5e942c56777fb6984 in 1680ms, sequenceid=317, compaction requested=true 2024-12-05T11:16:03,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a5726c82a42741c3adbf57a4a49bcb13, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/70f5c3e836c54ca181802ff0c6de2c15, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/2bc7a3aee0f945c1ad54e12131bcb4ad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5fd96820865d4693828c1787b9600eaa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a780e9949cbe432298e52c03a55ad798, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177] to archive 2024-12-05T11:16:03,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:16:03,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/e01364bbb2834f92b26bbf3ab1df234e 2024-12-05T11:16:03,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/d1abc4c7874d48328f005fe28135149c 2024-12-05T11:16:03,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9c80654fd43b46b79f040171f1c10830 2024-12-05T11:16:03,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3c991d7970ef46fd986c18c28bcce033 2024-12-05T11:16:03,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/8c81cb346e8c4258969b1ac17bc4cc28 2024-12-05T11:16:03,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a5726c82a42741c3adbf57a4a49bcb13 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a5726c82a42741c3adbf57a4a49bcb13 2024-12-05T11:16:03,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5e423570cd7e4801bd424e886b30572a 2024-12-05T11:16:03,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9d631ce5c3884806bd7f0b9b97e457a6 2024-12-05T11:16:03,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/70f5c3e836c54ca181802ff0c6de2c15 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/70f5c3e836c54ca181802ff0c6de2c15 2024-12-05T11:16:03,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/813bf0fbe5224e4ebe890e94af98d711 2024-12-05T11:16:03,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/325f9d7b933c4227b25703847e6c6bfa 2024-12-05T11:16:03,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/fa53a6cec65643e69f3184183de8d9c8 2024-12-05T11:16:03,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/2bc7a3aee0f945c1ad54e12131bcb4ad to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/2bc7a3aee0f945c1ad54e12131bcb4ad 2024-12-05T11:16:03,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/9a409d82881640c28ad0f5b9621ccdd9 2024-12-05T11:16:03,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/b0c915c06b294ae6be1b14c33ebc787c 2024-12-05T11:16:03,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5fd96820865d4693828c1787b9600eaa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/5fd96820865d4693828c1787b9600eaa 2024-12-05T11:16:03,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/7f14e866ef394e95908a53fd6f6d90be 2024-12-05T11:16:03,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/3172ed152a38419690ad4b7149a4faae 2024-12-05T11:16:03,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a780e9949cbe432298e52c03a55ad798 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/a780e9949cbe432298e52c03a55ad798 2024-12-05T11:16:03,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/f75e50ff3c994ff1b885f5338c02d69e 2024-12-05T11:16:03,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/63d5a49b7c5042edb8bfc0d85cc9e177 2024-12-05T11:16:03,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/54a2b50c52b2467ab1c590c2874b53c7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/831a50b9a3e748f1a4c272508d1971e4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/402548382b4d4d7eb1a639d3e2337ff3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/1831a89b06124faf86880ba947db07ac, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/d84aa5a6c4ff4b9d9007de68afea552d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/dac37781271c467c8489d9da1172189c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/4f46dc966a664ff8aab471bd05008b47, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c28fe208d86f4f2f8216918ab84b76fd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/76e9650edea0445f8762fba5e0c32566, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f43bd0e499c04c5290560f692673d94e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f7e041d599c64ff298cd7721c00b893f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/12b996ce4be448a787fe54b12b7f0b5a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/0b57c191961d44ba9e0b8ff7c13ac4df, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/b33c981fce854a80b81d0bd879b13e7d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/9ba7e0bdd1e24bff8188b341ece446c0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/90285905268840d593222525d3677557, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/34211c3358ba46038ec1cb293687ab6e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/59576d5ecfb64896bbb5d5f5a9d18cec, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/544ea01f5a8f45a99dfa391efe16cd17, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/e4f5dab24869435980d325f81813d2bb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/60b369ee97894a5687850e82085814a6] to archive 2024-12-05T11:16:03,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:16:03,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/54a2b50c52b2467ab1c590c2874b53c7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/54a2b50c52b2467ab1c590c2874b53c7 2024-12-05T11:16:03,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/831a50b9a3e748f1a4c272508d1971e4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/831a50b9a3e748f1a4c272508d1971e4 2024-12-05T11:16:03,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/402548382b4d4d7eb1a639d3e2337ff3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/402548382b4d4d7eb1a639d3e2337ff3 2024-12-05T11:16:03,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/1831a89b06124faf86880ba947db07ac to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/1831a89b06124faf86880ba947db07ac 2024-12-05T11:16:03,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/d84aa5a6c4ff4b9d9007de68afea552d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/d84aa5a6c4ff4b9d9007de68afea552d 2024-12-05T11:16:03,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/dac37781271c467c8489d9da1172189c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/dac37781271c467c8489d9da1172189c 2024-12-05T11:16:03,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/4f46dc966a664ff8aab471bd05008b47 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/4f46dc966a664ff8aab471bd05008b47 2024-12-05T11:16:03,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c28fe208d86f4f2f8216918ab84b76fd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c28fe208d86f4f2f8216918ab84b76fd 2024-12-05T11:16:03,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/76e9650edea0445f8762fba5e0c32566 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/76e9650edea0445f8762fba5e0c32566 2024-12-05T11:16:03,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f43bd0e499c04c5290560f692673d94e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f43bd0e499c04c5290560f692673d94e 2024-12-05T11:16:03,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f7e041d599c64ff298cd7721c00b893f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/f7e041d599c64ff298cd7721c00b893f 2024-12-05T11:16:03,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/12b996ce4be448a787fe54b12b7f0b5a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/12b996ce4be448a787fe54b12b7f0b5a 2024-12-05T11:16:03,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/0b57c191961d44ba9e0b8ff7c13ac4df to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/0b57c191961d44ba9e0b8ff7c13ac4df 2024-12-05T11:16:03,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/b33c981fce854a80b81d0bd879b13e7d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/b33c981fce854a80b81d0bd879b13e7d 2024-12-05T11:16:03,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/9ba7e0bdd1e24bff8188b341ece446c0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/9ba7e0bdd1e24bff8188b341ece446c0 2024-12-05T11:16:03,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/90285905268840d593222525d3677557 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/90285905268840d593222525d3677557 2024-12-05T11:16:03,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/34211c3358ba46038ec1cb293687ab6e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/34211c3358ba46038ec1cb293687ab6e 2024-12-05T11:16:03,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/59576d5ecfb64896bbb5d5f5a9d18cec to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/59576d5ecfb64896bbb5d5f5a9d18cec 2024-12-05T11:16:03,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/544ea01f5a8f45a99dfa391efe16cd17 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/544ea01f5a8f45a99dfa391efe16cd17 2024-12-05T11:16:03,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/e4f5dab24869435980d325f81813d2bb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/e4f5dab24869435980d325f81813d2bb 2024-12-05T11:16:03,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/60b369ee97894a5687850e82085814a6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/60b369ee97894a5687850e82085814a6 2024-12-05T11:16:03,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/e3c820d4992f4800ae7af76e4145d152, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/c2d549ec396b488e9c7d9963d68d594d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/cc869189f8fc422d89af807f11491fe8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/becbc51dbac043c4bb898e39326322a1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/49424104a6de45738ea799461c89339b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9874b67940be4a34b680fab3469bdd7b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/1d9d11630a1c406f8743f3e46a775f66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/60a35f245340421e9e740e3bcdca3fb4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/048fcf800c1349a88a9bf750afa350a2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/77a39fd4329c4d87b2f9d0337ff83f9d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/3178a0c06deb47fd85dd34284bc17cf1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/7f4d3ec5ec22432385108c7ae45f08fa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/8ad657b4d93a4e4cab792bf95d93bcdb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/0d783ce99b364470b33d98802cc2abc2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/5c5ef8642cc64bccbea635763ce157a4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/52237e65360d427f812000cd617bd1f3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/a80c09f72d26494bbfb8d639cbb73d5c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/2d2a61c8a1274e5894b8903b9f664b73, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/84f43960116e40c5b7de38ec0faaf490, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/51e5c21d769049b3a7d6b35d152e19ea, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/aed32ac323f9430796250bfe28cd7ba1] to archive 2024-12-05T11:16:03,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:16:03,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/e3c820d4992f4800ae7af76e4145d152 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/e3c820d4992f4800ae7af76e4145d152 2024-12-05T11:16:03,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/c2d549ec396b488e9c7d9963d68d594d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/c2d549ec396b488e9c7d9963d68d594d 2024-12-05T11:16:03,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/cc869189f8fc422d89af807f11491fe8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/cc869189f8fc422d89af807f11491fe8 2024-12-05T11:16:03,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/becbc51dbac043c4bb898e39326322a1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/becbc51dbac043c4bb898e39326322a1 2024-12-05T11:16:03,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/49424104a6de45738ea799461c89339b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/49424104a6de45738ea799461c89339b 2024-12-05T11:16:03,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9874b67940be4a34b680fab3469bdd7b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9874b67940be4a34b680fab3469bdd7b 2024-12-05T11:16:03,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/1d9d11630a1c406f8743f3e46a775f66 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/1d9d11630a1c406f8743f3e46a775f66 2024-12-05T11:16:03,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/60a35f245340421e9e740e3bcdca3fb4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/60a35f245340421e9e740e3bcdca3fb4 2024-12-05T11:16:03,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/048fcf800c1349a88a9bf750afa350a2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/048fcf800c1349a88a9bf750afa350a2 2024-12-05T11:16:03,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/77a39fd4329c4d87b2f9d0337ff83f9d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/77a39fd4329c4d87b2f9d0337ff83f9d 2024-12-05T11:16:03,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/3178a0c06deb47fd85dd34284bc17cf1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/3178a0c06deb47fd85dd34284bc17cf1 2024-12-05T11:16:03,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/7f4d3ec5ec22432385108c7ae45f08fa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/7f4d3ec5ec22432385108c7ae45f08fa 2024-12-05T11:16:03,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/8ad657b4d93a4e4cab792bf95d93bcdb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/8ad657b4d93a4e4cab792bf95d93bcdb 2024-12-05T11:16:03,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/0d783ce99b364470b33d98802cc2abc2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/0d783ce99b364470b33d98802cc2abc2 2024-12-05T11:16:03,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/5c5ef8642cc64bccbea635763ce157a4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/5c5ef8642cc64bccbea635763ce157a4 2024-12-05T11:16:03,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/52237e65360d427f812000cd617bd1f3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/52237e65360d427f812000cd617bd1f3 2024-12-05T11:16:03,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/a80c09f72d26494bbfb8d639cbb73d5c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/a80c09f72d26494bbfb8d639cbb73d5c 2024-12-05T11:16:03,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/2d2a61c8a1274e5894b8903b9f664b73 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/2d2a61c8a1274e5894b8903b9f664b73 2024-12-05T11:16:03,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/84f43960116e40c5b7de38ec0faaf490 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/84f43960116e40c5b7de38ec0faaf490 2024-12-05T11:16:03,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/51e5c21d769049b3a7d6b35d152e19ea to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/51e5c21d769049b3a7d6b35d152e19ea 2024-12-05T11:16:03,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/aed32ac323f9430796250bfe28cd7ba1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/aed32ac323f9430796250bfe28cd7ba1 2024-12-05T11:16:03,581 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/recovered.edits/320.seqid, newMaxSeqId=320, maxSeqId=4 2024-12-05T11:16:03,581 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. 2024-12-05T11:16:03,581 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for f72a0ecfe0b917c5e942c56777fb6984: Waiting for close lock at 1733397361845Running coprocessor pre-close hooks at 1733397361845Disabling compacts and flushes for region at 1733397361845Disabling writes for close at 1733397361845Obtaining lock to block concurrent updates at 1733397361845Preparing flush snapshotting stores in f72a0ecfe0b917c5e942c56777fb6984 at 1733397361845Finished memstore snapshotting TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733397361846 (+1 ms)Flushing stores of TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984. at 1733397361846Flushing f72a0ecfe0b917c5e942c56777fb6984/A: creating writer at 1733397361846Flushing mob file f72a0ecfe0b917c5e942c56777fb6984/A: appending metadata at 1733397361850 (+4 ms)Flushing mob file f72a0ecfe0b917c5e942c56777fb6984/A: closing flushed file at 1733397361850Flushing f72a0ecfe0b917c5e942c56777fb6984/A: appending metadata at 1733397362257 (+407 ms)Flushing f72a0ecfe0b917c5e942c56777fb6984/A: closing flushed file at 1733397362257Flushing f72a0ecfe0b917c5e942c56777fb6984/B: creating writer at 1733397362669 (+412 ms)Flushing f72a0ecfe0b917c5e942c56777fb6984/B: appending metadata at 1733397362674 (+5 ms)Flushing f72a0ecfe0b917c5e942c56777fb6984/B: closing flushed file at 1733397362674Flushing f72a0ecfe0b917c5e942c56777fb6984/C: creating writer at 1733397363092 (+418 ms)Flushing f72a0ecfe0b917c5e942c56777fb6984/C: appending metadata at 1733397363096 (+4 ms)Flushing f72a0ecfe0b917c5e942c56777fb6984/C: closing flushed file at 1733397363096Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c274a79: reopening flushed file at 1733397363509 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@445bf1d4: reopening flushed file at 1733397363516 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36dd0162: reopening flushed file at 1733397363521 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f72a0ecfe0b917c5e942c56777fb6984 in 1680ms, sequenceid=317, compaction requested=true at 1733397363525 (+4 ms)Writing region close event to WAL at 1733397363578 (+53 ms)Running coprocessor post-close hooks at 1733397363581 (+3 ms)Closed at 1733397363581 2024-12-05T11:16:03,583 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,584 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=f72a0ecfe0b917c5e942c56777fb6984, regionState=CLOSED 2024-12-05T11:16:03,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:16:03,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-05T11:16:03,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure f72a0ecfe0b917c5e942c56777fb6984, server=6706a97ae65f,42931,1733397243639 in 1.8930 sec 2024-12-05T11:16:03,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-05T11:16:03,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f72a0ecfe0b917c5e942c56777fb6984, UNASSIGN in 1.8990 sec 2024-12-05T11:16:03,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-05T11:16:03,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9040 sec 2024-12-05T11:16:03,594 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397363593"}]},"ts":"1733397363593"} 2024-12-05T11:16:03,595 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:16:03,595 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:16:03,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-12-05T11:16:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T11:16:03,839 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:03,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:16:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,842 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:16:03,843 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,845 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,848 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/recovered.edits] 2024-12-05T11:16:03,851 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/89d06a633cb14f02a935cdb5c46ba43d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/89d06a633cb14f02a935cdb5c46ba43d 2024-12-05T11:16:03,852 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/c0e3679629ee464aa46b871bc5781c1f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/c0e3679629ee464aa46b871bc5781c1f 2024-12-05T11:16:03,853 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/ffaa2e30211f4799854a7a59381a89a8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/A/ffaa2e30211f4799854a7a59381a89a8 2024-12-05T11:16:03,856 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/2f80d3288b764b54a31c267b831b4f86 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/2f80d3288b764b54a31c267b831b4f86 2024-12-05T11:16:03,857 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/6c91e5561fd54a9eb0bc9912d4005faf to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/6c91e5561fd54a9eb0bc9912d4005faf 2024-12-05T11:16:03,858 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c61325bab0df46e784a5e8a76b90748f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/B/c61325bab0df46e784a5e8a76b90748f 2024-12-05T11:16:03,860 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/54c6a8926bf24805a614a456549c930c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/54c6a8926bf24805a614a456549c930c 2024-12-05T11:16:03,861 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/56e0f0ad6ff0405d963466673e16c03d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/56e0f0ad6ff0405d963466673e16c03d 2024-12-05T11:16:03,862 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9eb2f17e662840d3ab612c3d9e35c664 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/C/9eb2f17e662840d3ab612c3d9e35c664 2024-12-05T11:16:03,865 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/recovered.edits/320.seqid to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984/recovered.edits/320.seqid 2024-12-05T11:16:03,865 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,865 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:16:03,865 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:16:03,866 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T11:16:03,868 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050296223233fa4bcc86a962eff32e520e_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050296223233fa4bcc86a962eff32e520e_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,869 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120502fd2924bb2f4cccbb1483c93374eeb5_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120502fd2924bb2f4cccbb1483c93374eeb5_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,869 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205056f5f40d02947d797c1898d0c12784b_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205056f5f40d02947d797c1898d0c12784b_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,870 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051cd87feb1aa340539c0ebebbb763fc59_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051cd87feb1aa340539c0ebebbb763fc59_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,871 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055156011831594f2d98777c75f6037fe1_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055156011831594f2d98777c75f6037fe1_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,871 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055910495191a1467db201641fd3893393_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055910495191a1467db201641fd3893393_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,872 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056dcc16b994c64c779525a1e933c8e0a6_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056dcc16b994c64c779525a1e933c8e0a6_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,873 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120571d6df7eac4f4c7fa9bcc2a3d0777ef3_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120571d6df7eac4f4c7fa9bcc2a3d0777ef3_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,874 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ce5c20da05d4d9fae8d445daf634cab_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ce5c20da05d4d9fae8d445daf634cab_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,874 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120599a49aabc9034b21ad7fbbfec69a78e5_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120599a49aabc9034b21ad7fbbfec69a78e5_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,875 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a60ae375fd0a43b28272cab093aa6b4f_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a60ae375fd0a43b28272cab093aa6b4f_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,876 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a745612dbfce4ad0bd8884a22e223ce0_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a745612dbfce4ad0bd8884a22e223ce0_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,876 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205aeb1a165bf904af2bf098a00424d6336_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205aeb1a165bf904af2bf098a00424d6336_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,877 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1361893d2b64f8890665d443a7044f9_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c1361893d2b64f8890665d443a7044f9_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,878 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cf3e11a38682439bb7147f36b7826560_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cf3e11a38682439bb7147f36b7826560_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,879 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d7960bb8c3f24f8eb1f9e311407f174d_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d7960bb8c3f24f8eb1f9e311407f174d_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,880 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ea1aa7e6064e4d949cb1d24b8ed9e69b_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ea1aa7e6064e4d949cb1d24b8ed9e69b_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,880 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fe070065fe4547dcb50d7925de4ef73a_f72a0ecfe0b917c5e942c56777fb6984 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fe070065fe4547dcb50d7925de4ef73a_f72a0ecfe0b917c5e942c56777fb6984 2024-12-05T11:16:03,881 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:16:03,883 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,884 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:16:03,886 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:16:03,887 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,887 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:16:03,887 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733397363887"}]},"ts":"9223372036854775807"} 2024-12-05T11:16:03,889 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:16:03,889 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f72a0ecfe0b917c5e942c56777fb6984, NAME => 'TestAcidGuarantees,,1733397335167.f72a0ecfe0b917c5e942c56777fb6984.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:16:03,889 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:16:03,889 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733397363889"}]},"ts":"9223372036854775807"} 2024-12-05T11:16:03,890 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:16:03,891 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-12-05T11:16:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T11:16:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:16:03,955 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:03,967 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 250), OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 379) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6092 (was 6794) 2024-12-05T11:16:03,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=6092 2024-12-05T11:16:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:16:03,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:16:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:03,979 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:16:03,979 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:03,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-12-05T11:16:03,980 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:16:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T11:16:03,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742230_1406 (size=1061) 2024-12-05T11:16:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T11:16:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T11:16:04,386 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:16:04,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742231_1407 (size=53) 2024-12-05T11:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T11:16:04,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:16:04,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e19ca153de371b50b73536c04e3fa097, disabling compactions & flushes 2024-12-05T11:16:04,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:04,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:04,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. after waiting 0 ms 2024-12-05T11:16:04,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:04,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:04,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e19ca153de371b50b73536c04e3fa097: Waiting for close lock at 1733397364804Disabling compacts and flushes for region at 1733397364804Disabling writes for close at 1733397364805 (+1 ms)Writing region close event to WAL at 1733397364805Closed at 1733397364805 2024-12-05T11:16:04,807 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:16:04,807 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733397364807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733397364807"}]},"ts":"1733397364807"} 2024-12-05T11:16:04,810 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:16:04,811 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:16:04,811 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397364811"}]},"ts":"1733397364811"} 2024-12-05T11:16:04,813 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:16:04,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, ASSIGN}] 2024-12-05T11:16:04,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, ASSIGN 2024-12-05T11:16:04,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, ASSIGN; state=OFFLINE, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=false 2024-12-05T11:16:04,968 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=e19ca153de371b50b73536c04e3fa097, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:04,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, ASSIGN because future has completed 2024-12-05T11:16:04,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:16:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T11:16:05,126 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:05,126 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:16:05,127 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,127 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:16:05,127 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,127 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,129 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,131 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:05,131 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e19ca153de371b50b73536c04e3fa097 columnFamilyName A 2024-12-05T11:16:05,131 DEBUG [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:05,133 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.HStore(327): Store=e19ca153de371b50b73536c04e3fa097/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:05,133 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,135 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:05,135 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e19ca153de371b50b73536c04e3fa097 columnFamilyName B 2024-12-05T11:16:05,135 DEBUG [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:05,135 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.HStore(327): Store=e19ca153de371b50b73536c04e3fa097/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:05,136 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,137 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:05,137 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e19ca153de371b50b73536c04e3fa097 columnFamilyName C 2024-12-05T11:16:05,137 DEBUG [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:05,138 INFO [StoreOpener-e19ca153de371b50b73536c04e3fa097-1 {}] regionserver.HStore(327): Store=e19ca153de371b50b73536c04e3fa097/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:05,138 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:05,138 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,139 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,139 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,144 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,144 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,145 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:16:05,148 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,151 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:16:05,152 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened e19ca153de371b50b73536c04e3fa097; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66319636, jitterRate=-0.011760413646697998}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:16:05,152 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:05,153 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for e19ca153de371b50b73536c04e3fa097: Running coprocessor pre-open hook at 1733397365127Writing region info on filesystem at 1733397365127Initializing all the Stores at 1733397365129 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397365129Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397365129Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397365129Cleaning up temporary data from old regions at 1733397365144 (+15 ms)Running coprocessor post-open hooks at 1733397365152 (+8 ms)Region opened successfully at 1733397365153 (+1 ms) 2024-12-05T11:16:05,154 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., pid=110, masterSystemTime=1733397365123 2024-12-05T11:16:05,156 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:05,156 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:05,157 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=e19ca153de371b50b73536c04e3fa097, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:05,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:16:05,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-05T11:16:05,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 in 190 msec 2024-12-05T11:16:05,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-05T11:16:05,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, ASSIGN in 353 msec 2024-12-05T11:16:05,173 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:16:05,174 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397365173"}]},"ts":"1733397365173"} 2024-12-05T11:16:05,176 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:16:05,178 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:16:05,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-12-05T11:16:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T11:16:06,129 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:06,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309953ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,131 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f35db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,133 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0f2952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,134 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,137 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:16:06,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:16:06,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ea253e9 2024-12-05T11:16:06,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:16:06,139 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:16:06,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135be63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,141 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcfdb95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,143 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cba04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432aef64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ec0e21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,147 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe1affe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a177044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,149 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ba14a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,151 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@660a5406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbbba3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,153 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcf2578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,155 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17147f98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd9b86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,157 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf89975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c9af20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48bc9769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,161 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbc993f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,162 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f63b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460a8c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,165 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711321b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad93139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b76d744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,168 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3728be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,170 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330ffc4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694dc39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,172 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a65482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,174 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224b4de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f57cd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:06,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:06,177 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8bf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:06,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bd8103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:06,181 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,181 DEBUG [Thread-1855 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,181 DEBUG [Thread-1850 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,182 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,182 DEBUG [Thread-1846 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,185 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,186 DEBUG [Thread-1852 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,190 DEBUG [Thread-1852 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,190 DEBUG [Thread-1844 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,191 DEBUG [Thread-1852 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,199 DEBUG [Thread-1848 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:06,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:16:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:06,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:06,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:06,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:06,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:06,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,216 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-05T11:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:16:06,220 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:06,220 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:06,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:06,223 DEBUG [Thread-1857 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:06,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2] 2024-12-05T11:16:06,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397366220, max timestamp=1733397366235Num uniques 859; 2024-12-05T11:16:06,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397366220, max timestamp=1733397366235Num uniques 859; 2024-12-05T11:16:06,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:06,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:06,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397366220, max timestamp=1733397366235Num uniques 859; 2024-12-05T11:16:06,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:06,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4898c899c266471eb426470aa9ceab8a is 50, key is test_row_0/A:col10/1733397366197/Put/seqid=0 2024-12-05T11:16:06,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742232_1408 (size=12001) 2024-12-05T11:16:06,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4898c899c266471eb426470aa9ceab8a 2024-12-05T11:16:06,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/1790b222c2ca401ca16699f103860e66 is 50, key is test_row_0/B:col10/1733397366197/Put/seqid=0 2024-12-05T11:16:06,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742233_1409 (size=12001) 2024-12-05T11:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:16:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397426315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397426316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397426329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397426347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397426347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T11:16:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397426448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397426451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397426469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397426469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397426469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T11:16:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:16:06,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397426659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397426668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T11:16:06,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:06,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397426679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397426679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397426679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:06,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/1790b222c2ca401ca16699f103860e66 2024-12-05T11:16:06,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/76b862881dfc41c8a642994e5b730d22 is 50, key is test_row_0/C:col10/1733397366197/Put/seqid=0 2024-12-05T11:16:06,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742234_1410 (size=12001) 2024-12-05T11:16:06,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/76b862881dfc41c8a642994e5b730d22 2024-12-05T11:16:06,794 WARN [regionserver/6706a97ae65f:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-12-05T11:16:06,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4898c899c266471eb426470aa9ceab8a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4898c899c266471eb426470aa9ceab8a 2024-12-05T11:16:06,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4898c899c266471eb426470aa9ceab8a, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T11:16:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/1790b222c2ca401ca16699f103860e66 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1790b222c2ca401ca16699f103860e66 2024-12-05T11:16:06,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1790b222c2ca401ca16699f103860e66, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T11:16:06,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/76b862881dfc41c8a642994e5b730d22 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/76b862881dfc41c8a642994e5b730d22 2024-12-05T11:16:06,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/76b862881dfc41c8a642994e5b730d22, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T11:16:06,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for e19ca153de371b50b73536c04e3fa097 in 618ms, sequenceid=13, compaction requested=false 2024-12-05T11:16:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:06,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T11:16:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:06,837 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-05T11:16:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:06,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/e414242a7eba4759a1896601350c3e6b is 50, key is test_row_0/A:col10/1733397366315/Put/seqid=0 2024-12-05T11:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:16:06,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742235_1411 (size=12001) 2024-12-05T11:16:06,862 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/e414242a7eba4759a1896601350c3e6b 2024-12-05T11:16:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/96938823e50540eca52a8354df255506 is 50, key is test_row_0/B:col10/1733397366315/Put/seqid=0 2024-12-05T11:16:06,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742236_1412 (size=12001) 2024-12-05T11:16:06,911 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/96938823e50540eca52a8354df255506 2024-12-05T11:16:06,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/c767340d9fdf40249ab6d614c3b1aaac is 50, key is test_row_0/C:col10/1733397366315/Put/seqid=0 2024-12-05T11:16:06,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742237_1413 (size=12001) 2024-12-05T11:16:06,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397427003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397427005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397427006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397427007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397427009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397427118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397427119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397427119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397427128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397427128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397427338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397427338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397427339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397427339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397427339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,347 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/c767340d9fdf40249ab6d614c3b1aaac 2024-12-05T11:16:07,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/e414242a7eba4759a1896601350c3e6b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/e414242a7eba4759a1896601350c3e6b 2024-12-05T11:16:07,353 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/e414242a7eba4759a1896601350c3e6b, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T11:16:07,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/96938823e50540eca52a8354df255506 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/96938823e50540eca52a8354df255506 2024-12-05T11:16:07,358 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/96938823e50540eca52a8354df255506, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T11:16:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:16:07,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/c767340d9fdf40249ab6d614c3b1aaac as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/c767340d9fdf40249ab6d614c3b1aaac 2024-12-05T11:16:07,361 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/c767340d9fdf40249ab6d614c3b1aaac, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T11:16:07,361 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for e19ca153de371b50b73536c04e3fa097 in 524ms, sequenceid=47, compaction requested=false 2024-12-05T11:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-05T11:16:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-05T11:16:07,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-05T11:16:07,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1430 sec 2024-12-05T11:16:07,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.1490 sec 2024-12-05T11:16:07,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:16:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:07,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1cb0477d7af64747ba4f81d011cef9c9 is 50, key is test_row_0/A:col10/1733397367659/Put/seqid=0 2024-12-05T11:16:07,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742238_1414 (size=14341) 2024-12-05T11:16:07,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1cb0477d7af64747ba4f81d011cef9c9 2024-12-05T11:16:07,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/c42a058672504d3ea87b1b71a5597c7b is 50, key is test_row_0/B:col10/1733397367659/Put/seqid=0 2024-12-05T11:16:07,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397427710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397427713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397427714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397427715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397427715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742239_1415 (size=12001) 2024-12-05T11:16:07,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/c42a058672504d3ea87b1b71a5597c7b 2024-12-05T11:16:07,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/6259bf3c9f424d5182b5b96a47e45ce1 is 50, key is test_row_0/C:col10/1733397367659/Put/seqid=0 2024-12-05T11:16:07,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742240_1416 (size=12001) 2024-12-05T11:16:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397427838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397427838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397427839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397427839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397427839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397428049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397428049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397428049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397428049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397428058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/6259bf3c9f424d5182b5b96a47e45ce1 2024-12-05T11:16:08,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1cb0477d7af64747ba4f81d011cef9c9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1cb0477d7af64747ba4f81d011cef9c9 2024-12-05T11:16:08,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1cb0477d7af64747ba4f81d011cef9c9, entries=200, sequenceid=60, filesize=14.0 K 2024-12-05T11:16:08,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/c42a058672504d3ea87b1b71a5597c7b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/c42a058672504d3ea87b1b71a5597c7b 2024-12-05T11:16:08,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/c42a058672504d3ea87b1b71a5597c7b, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:16:08,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/6259bf3c9f424d5182b5b96a47e45ce1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/6259bf3c9f424d5182b5b96a47e45ce1 2024-12-05T11:16:08,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/6259bf3c9f424d5182b5b96a47e45ce1, entries=150, sequenceid=60, filesize=11.7 K 2024-12-05T11:16:08,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e19ca153de371b50b73536c04e3fa097 in 565ms, sequenceid=60, compaction requested=true 2024-12-05T11:16:08,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:08,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:08,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:08,227 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:08,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:08,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:08,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:08,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:08,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:08,227 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:08,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:08,227 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:08,227 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:08,227 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,227 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,228 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4898c899c266471eb426470aa9ceab8a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/e414242a7eba4759a1896601350c3e6b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1cb0477d7af64747ba4f81d011cef9c9] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=37.4 K 2024-12-05T11:16:08,228 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1790b222c2ca401ca16699f103860e66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/96938823e50540eca52a8354df255506, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/c42a058672504d3ea87b1b71a5597c7b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=35.2 K 2024-12-05T11:16:08,228 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4898c899c266471eb426470aa9ceab8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733397366197 2024-12-05T11:16:08,228 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1790b222c2ca401ca16699f103860e66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733397366197 2024-12-05T11:16:08,228 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting e414242a7eba4759a1896601350c3e6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733397366314 2024-12-05T11:16:08,228 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 96938823e50540eca52a8354df255506, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733397366314 2024-12-05T11:16:08,228 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c42a058672504d3ea87b1b71a5597c7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397366994 2024-12-05T11:16:08,228 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb0477d7af64747ba4f81d011cef9c9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397366994 2024-12-05T11:16:08,234 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:08,234 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/93f9ccad03304bbbae62e30afa861388 is 50, key is test_row_0/A:col10/1733397367659/Put/seqid=0 2024-12-05T11:16:08,237 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:08,238 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3b0c9b36f70d4c958af0818650372100 is 50, key is test_row_0/B:col10/1733397367659/Put/seqid=0 2024-12-05T11:16:08,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742241_1417 (size=12104) 2024-12-05T11:16:08,249 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/93f9ccad03304bbbae62e30afa861388 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/93f9ccad03304bbbae62e30afa861388 2024-12-05T11:16:08,253 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into 93f9ccad03304bbbae62e30afa861388(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:08,253 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:08,253 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=13, startTime=1733397368226; duration=0sec 2024-12-05T11:16:08,253 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:08,253 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:08,253 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:08,254 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:08,254 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:08,254 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,255 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/76b862881dfc41c8a642994e5b730d22, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/c767340d9fdf40249ab6d614c3b1aaac, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/6259bf3c9f424d5182b5b96a47e45ce1] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=35.2 K 2024-12-05T11:16:08,255 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b862881dfc41c8a642994e5b730d22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733397366197 2024-12-05T11:16:08,255 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c767340d9fdf40249ab6d614c3b1aaac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733397366314 2024-12-05T11:16:08,255 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6259bf3c9f424d5182b5b96a47e45ce1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397366994 2024-12-05T11:16:08,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742242_1418 (size=12104) 2024-12-05T11:16:08,267 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3b0c9b36f70d4c958af0818650372100 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3b0c9b36f70d4c958af0818650372100 2024-12-05T11:16:08,270 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:08,270 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/fbb27663cc0a46d5829f217296f32cb8 is 50, key is test_row_0/C:col10/1733397367659/Put/seqid=0 2024-12-05T11:16:08,272 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into 3b0c9b36f70d4c958af0818650372100(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:08,272 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:08,272 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=13, startTime=1733397368227; duration=0sec 2024-12-05T11:16:08,272 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:08,272 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:08,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742243_1419 (size=12104) 2024-12-05T11:16:08,287 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/fbb27663cc0a46d5829f217296f32cb8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/fbb27663cc0a46d5829f217296f32cb8 2024-12-05T11:16:08,291 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into fbb27663cc0a46d5829f217296f32cb8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:08,291 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:08,291 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=13, startTime=1733397368227; duration=0sec 2024-12-05T11:16:08,291 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:08,291 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:08,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T11:16:08,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:08,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:08,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:08,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:08,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:08,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:08,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/77ad59f392ed4f90b4902cc7c8e2f922 is 50, key is test_row_0/A:col10/1733397368362/Put/seqid=0 2024-12-05T11:16:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T11:16:08,368 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:08,369 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-05T11:16:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:08,370 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:08,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742244_1420 (size=16681) 2024-12-05T11:16:08,371 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:08,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397428373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397428374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397428374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397428375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397428378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397428478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397428488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397428489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397428489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:08,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:08,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:08,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:08,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397428688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397428698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397428698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397428699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/77ad59f392ed4f90b4902cc7c8e2f922 2024-12-05T11:16:08,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/1697c1dbae9047dc9d1361e58b90200a is 50, key is test_row_0/B:col10/1733397368362/Put/seqid=0 2024-12-05T11:16:08,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742245_1421 (size=12001) 2024-12-05T11:16:08,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397428898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:08,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:08,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397429008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397429018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397429018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397429018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,097 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:16:09,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/1697c1dbae9047dc9d1361e58b90200a 2024-12-05T11:16:09,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/5707aa27a74d4a99951c685347faed14 is 50, key is test_row_0/C:col10/1733397368362/Put/seqid=0 2024-12-05T11:16:09,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742246_1422 (size=12001) 2024-12-05T11:16:09,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:09,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397429529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397429539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397429539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397429539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:09,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/5707aa27a74d4a99951c685347faed14 2024-12-05T11:16:09,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:09,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/77ad59f392ed4f90b4902cc7c8e2f922 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/77ad59f392ed4f90b4902cc7c8e2f922 2024-12-05T11:16:09,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/77ad59f392ed4f90b4902cc7c8e2f922, entries=250, sequenceid=85, filesize=16.3 K 2024-12-05T11:16:09,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/1697c1dbae9047dc9d1361e58b90200a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1697c1dbae9047dc9d1361e58b90200a 2024-12-05T11:16:09,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1697c1dbae9047dc9d1361e58b90200a, entries=150, sequenceid=85, filesize=11.7 K 2024-12-05T11:16:09,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/5707aa27a74d4a99951c685347faed14 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5707aa27a74d4a99951c685347faed14 2024-12-05T11:16:09,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5707aa27a74d4a99951c685347faed14, entries=150, sequenceid=85, filesize=11.7 K 2024-12-05T11:16:09,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for e19ca153de371b50b73536c04e3fa097 in 1264ms, sequenceid=85, compaction requested=false 2024-12-05T11:16:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:09,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T11:16:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:09,755 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:16:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1b32933ba0f24abd84272a7b1c8947d9 is 50, key is test_row_0/A:col10/1733397368372/Put/seqid=0 2024-12-05T11:16:09,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742247_1423 (size=12001) 2024-12-05T11:16:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:09,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397370012, max timestamp=1733397370041Num uniques 859; 2024-12-05T11:16:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397370012, max timestamp=1733397370041Num uniques 859; 2024-12-05T11:16:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397370012, max timestamp=1733397370041Num uniques 859; 2024-12-05T11:16:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397430200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,208 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1b32933ba0f24abd84272a7b1c8947d9 2024-12-05T11:16:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/442a0ed4cfdf44bc888a338f5f653b65 is 50, key is test_row_0/B:col10/1733397368372/Put/seqid=0 2024-12-05T11:16:10,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742248_1424 (size=12001) 2024-12-05T11:16:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397430319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397430528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397430548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,556 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397430559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397430559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397430559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,565 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,565 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,566 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:10,617 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/442a0ed4cfdf44bc888a338f5f653b65 2024-12-05T11:16:10,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f0c8261dac1d4b4f84b56d68de214758 is 50, key is test_row_0/C:col10/1733397368372/Put/seqid=0 2024-12-05T11:16:10,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742249_1425 (size=12001) 2024-12-05T11:16:10,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397430848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:11,047 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f0c8261dac1d4b4f84b56d68de214758 2024-12-05T11:16:11,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1b32933ba0f24abd84272a7b1c8947d9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1b32933ba0f24abd84272a7b1c8947d9 2024-12-05T11:16:11,054 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1b32933ba0f24abd84272a7b1c8947d9, entries=150, sequenceid=95, filesize=11.7 K 2024-12-05T11:16:11,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/442a0ed4cfdf44bc888a338f5f653b65 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/442a0ed4cfdf44bc888a338f5f653b65 2024-12-05T11:16:11,057 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/442a0ed4cfdf44bc888a338f5f653b65, entries=150, sequenceid=95, filesize=11.7 K 2024-12-05T11:16:11,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f0c8261dac1d4b4f84b56d68de214758 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0c8261dac1d4b4f84b56d68de214758 2024-12-05T11:16:11,063 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0c8261dac1d4b4f84b56d68de214758, entries=150, sequenceid=95, filesize=11.7 K 2024-12-05T11:16:11,064 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e19ca153de371b50b73536c04e3fa097 in 1308ms, sequenceid=95, compaction requested=true 2024-12-05T11:16:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-05T11:16:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-05T11:16:11,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-05T11:16:11,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6940 sec 2024-12-05T11:16:11,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.6980 sec 2024-12-05T11:16:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:11,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-05T11:16:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:16:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:16:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:16:11,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1d52b4d1b0b64145b9b758f022af3545 is 50, key is test_row_0/A:col10/1733397370193/Put/seqid=0 2024-12-05T11:16:11,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742250_1426 (size=14541) 2024-12-05T11:16:11,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397431413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:11,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397431528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397431749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:11,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1d52b4d1b0b64145b9b758f022af3545 2024-12-05T11:16:11,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/f9f671314bc44f3bae98b698922d8958 is 50, key is test_row_0/B:col10/1733397370193/Put/seqid=0 2024-12-05T11:16:11,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742251_1427 (size=12151) 2024-12-05T11:16:12,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397432058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:12,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/f9f671314bc44f3bae98b698922d8958 2024-12-05T11:16:12,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/0ff0654997e34cde8c88c300af71a10c is 50, key is test_row_0/C:col10/1733397370193/Put/seqid=0 2024-12-05T11:16:12,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742252_1428 (size=12151) 2024-12-05T11:16:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T11:16:12,528 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:12,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-05T11:16:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:16:12,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:12,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:12,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397432568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:12,578 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397432578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397432579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,591 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:12,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/0ff0654997e34cde8c88c300af71a10c 2024-12-05T11:16:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397432588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:12,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397432588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:12,602 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_2' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,602 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:12,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/1d52b4d1b0b64145b9b758f022af3545 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1d52b4d1b0b64145b9b758f022af3545 2024-12-05T11:16:12,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1d52b4d1b0b64145b9b758f022af3545, entries=200, sequenceid=133, filesize=14.2 K 2024-12-05T11:16:12,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/f9f671314bc44f3bae98b698922d8958 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f9f671314bc44f3bae98b698922d8958 2024-12-05T11:16:12,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f9f671314bc44f3bae98b698922d8958, entries=150, sequenceid=133, filesize=11.9 K 2024-12-05T11:16:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/0ff0654997e34cde8c88c300af71a10c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0ff0654997e34cde8c88c300af71a10c 2024-12-05T11:16:12,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0ff0654997e34cde8c88c300af71a10c, entries=150, sequenceid=133, filesize=11.9 K 2024-12-05T11:16:12,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e19ca153de371b50b73536c04e3fa097 in 1258ms, sequenceid=133, compaction requested=true 2024-12-05T11:16:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:12,620 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:12,620 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:12,621 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:12,621 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55327 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:12,621 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:12,621 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:12,621 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:12,621 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:12,621 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/93f9ccad03304bbbae62e30afa861388, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/77ad59f392ed4f90b4902cc7c8e2f922, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1b32933ba0f24abd84272a7b1c8947d9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1d52b4d1b0b64145b9b758f022af3545] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=54.0 K 2024-12-05T11:16:12,621 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3b0c9b36f70d4c958af0818650372100, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1697c1dbae9047dc9d1361e58b90200a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/442a0ed4cfdf44bc888a338f5f653b65, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f9f671314bc44f3bae98b698922d8958] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=47.1 K 2024-12-05T11:16:12,621 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0c9b36f70d4c958af0818650372100, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397366994 2024-12-05T11:16:12,621 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f9ccad03304bbbae62e30afa861388, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397366994 2024-12-05T11:16:12,622 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1697c1dbae9047dc9d1361e58b90200a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733397367713 2024-12-05T11:16:12,622 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ad59f392ed4f90b4902cc7c8e2f922, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733397367694 2024-12-05T11:16:12,622 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b32933ba0f24abd84272a7b1c8947d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733397368372 2024-12-05T11:16:12,622 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 442a0ed4cfdf44bc888a338f5f653b65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733397368372 2024-12-05T11:16:12,622 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d52b4d1b0b64145b9b758f022af3545, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397370185 2024-12-05T11:16:12,622 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f671314bc44f3bae98b698922d8958, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397370185 2024-12-05T11:16:12,630 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:12,631 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/fae69641b42d492782740ad7906d562c is 50, key is test_row_0/B:col10/1733397370193/Put/seqid=0 2024-12-05T11:16:12,635 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:12,636 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/807f498965664cc59eb371576df91d53 is 50, key is test_row_0/A:col10/1733397370193/Put/seqid=0 2024-12-05T11:16:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:16:12,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742253_1429 (size=12391) 2024-12-05T11:16:12,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742254_1430 (size=12391) 2024-12-05T11:16:12,661 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/fae69641b42d492782740ad7906d562c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/fae69641b42d492782740ad7906d562c 2024-12-05T11:16:12,673 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into fae69641b42d492782740ad7906d562c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:12,673 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:12,673 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=12, startTime=1733397372620; duration=0sec 2024-12-05T11:16:12,673 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:12,673 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:12,673 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:12,674 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:12,674 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:12,674 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:12,674 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/fbb27663cc0a46d5829f217296f32cb8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5707aa27a74d4a99951c685347faed14, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0c8261dac1d4b4f84b56d68de214758, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0ff0654997e34cde8c88c300af71a10c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=47.1 K 2024-12-05T11:16:12,675 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb27663cc0a46d5829f217296f32cb8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733397366994 2024-12-05T11:16:12,675 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5707aa27a74d4a99951c685347faed14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733397367713 2024-12-05T11:16:12,675 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c8261dac1d4b4f84b56d68de214758, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733397368372 2024-12-05T11:16:12,675 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff0654997e34cde8c88c300af71a10c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397370185 2024-12-05T11:16:12,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T11:16:12,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:12,683 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T11:16:12,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/392f9a942215472687e4b354002712fb is 50, key is test_row_0/A:col10/1733397371405/Put/seqid=0 2024-12-05T11:16:12,691 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#360 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:12,691 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/83715bc8bbc74b1a9277ac32e5ea96e2 is 50, key is test_row_0/C:col10/1733397370193/Put/seqid=0 2024-12-05T11:16:12,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742255_1431 (size=12391) 2024-12-05T11:16:12,719 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/83715bc8bbc74b1a9277ac32e5ea96e2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/83715bc8bbc74b1a9277ac32e5ea96e2 2024-12-05T11:16:12,723 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into 83715bc8bbc74b1a9277ac32e5ea96e2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:12,723 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:12,723 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=12, startTime=1733397372620; duration=0sec 2024-12-05T11:16:12,723 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:12,723 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:12,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742256_1432 (size=12151) 2024-12-05T11:16:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:16:13,061 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/807f498965664cc59eb371576df91d53 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/807f498965664cc59eb371576df91d53 2024-12-05T11:16:13,065 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into 807f498965664cc59eb371576df91d53(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:13,065 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:13,066 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=12, startTime=1733397372619; duration=0sec 2024-12-05T11:16:13,066 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:13,066 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:13,127 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/392f9a942215472687e4b354002712fb 2024-12-05T11:16:13,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/e26de87f74854c93877caf6825222b7b is 50, key is test_row_0/B:col10/1733397371405/Put/seqid=0 2024-12-05T11:16:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742257_1433 (size=12151) 2024-12-05T11:16:13,146 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/e26de87f74854c93877caf6825222b7b 2024-12-05T11:16:13,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9c4a2f85fcac4d9598e7e5ee568bafa9 is 50, key is test_row_0/C:col10/1733397371405/Put/seqid=0 2024-12-05T11:16:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:16:13,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742258_1434 (size=12151) 2024-12-05T11:16:13,563 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9c4a2f85fcac4d9598e7e5ee568bafa9 2024-12-05T11:16:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/392f9a942215472687e4b354002712fb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/392f9a942215472687e4b354002712fb 2024-12-05T11:16:13,575 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/392f9a942215472687e4b354002712fb, entries=150, sequenceid=143, filesize=11.9 K 2024-12-05T11:16:13,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/e26de87f74854c93877caf6825222b7b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e26de87f74854c93877caf6825222b7b 2024-12-05T11:16:13,579 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e26de87f74854c93877caf6825222b7b, entries=150, sequenceid=143, filesize=11.9 K 2024-12-05T11:16:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9c4a2f85fcac4d9598e7e5ee568bafa9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9c4a2f85fcac4d9598e7e5ee568bafa9 2024-12-05T11:16:13,583 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9c4a2f85fcac4d9598e7e5ee568bafa9, entries=150, sequenceid=143, filesize=11.9 K 2024-12-05T11:16:13,584 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for e19ca153de371b50b73536c04e3fa097 in 901ms, sequenceid=143, compaction requested=false 2024-12-05T11:16:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-05T11:16:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-05T11:16:13,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-05T11:16:13,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0550 sec 2024-12-05T11:16:13,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0590 sec 2024-12-05T11:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:13,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:16:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:13,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/07e5adf8a28b4051b14c1a6f50aabda5 is 50, key is test_row_0/A:col10/1733397373631/Put/seqid=0 2024-12-05T11:16:13,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742259_1435 (size=14541) 2024-12-05T11:16:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T11:16:13,668 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:13,669 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-05T11:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:13,670 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:13,670 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:13,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:13,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397373676, max timestamp=1733397373749Num uniques 859; 2024-12-05T11:16:13,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397373676, max timestamp=1733397373749Num uniques 859; 2024-12-05T11:16:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397373676, max timestamp=1733397373749Num uniques 859; 2024-12-05T11:16:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:13,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:13,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:13,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397433849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397433968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:13,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:13,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:14,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/07e5adf8a28b4051b14c1a6f50aabda5 2024-12-05T11:16:14,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/9bbc91b7bcd241d09bfdba749a71bf84 is 50, key is test_row_0/B:col10/1733397373631/Put/seqid=0 2024-12-05T11:16:14,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742260_1436 (size=12151) 2024-12-05T11:16:14,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:14,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:14,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397434178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:14,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:14,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:14,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:14,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:14,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:14,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:14,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/9bbc91b7bcd241d09bfdba749a71bf84 2024-12-05T11:16:14,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/0985395cc83d4169a8601900cee891c2 is 50, key is test_row_0/C:col10/1733397373631/Put/seqid=0 2024-12-05T11:16:14,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742261_1437 (size=12151) 2024-12-05T11:16:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397434488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:14,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:14,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:14,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/0985395cc83d4169a8601900cee891c2 2024-12-05T11:16:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/07e5adf8a28b4051b14c1a6f50aabda5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/07e5adf8a28b4051b14c1a6f50aabda5 2024-12-05T11:16:14,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/07e5adf8a28b4051b14c1a6f50aabda5, entries=200, sequenceid=155, filesize=14.2 K 2024-12-05T11:16:14,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/9bbc91b7bcd241d09bfdba749a71bf84 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/9bbc91b7bcd241d09bfdba749a71bf84 2024-12-05T11:16:14,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/9bbc91b7bcd241d09bfdba749a71bf84, entries=150, sequenceid=155, filesize=11.9 K 2024-12-05T11:16:14,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/0985395cc83d4169a8601900cee891c2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0985395cc83d4169a8601900cee891c2 2024-12-05T11:16:14,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0985395cc83d4169a8601900cee891c2, entries=150, sequenceid=155, filesize=11.9 K 2024-12-05T11:16:14,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e19ca153de371b50b73536c04e3fa097 in 1279ms, sequenceid=155, compaction requested=true 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:14,911 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:14,911 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:14,912 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:14,912 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:14,912 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:14,912 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:14,912 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,912 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:14,912 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/fae69641b42d492782740ad7906d562c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e26de87f74854c93877caf6825222b7b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/9bbc91b7bcd241d09bfdba749a71bf84] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=35.8 K 2024-12-05T11:16:14,912 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/807f498965664cc59eb371576df91d53, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/392f9a942215472687e4b354002712fb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/07e5adf8a28b4051b14c1a6f50aabda5] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=38.2 K 2024-12-05T11:16:14,912 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting fae69641b42d492782740ad7906d562c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397370185 2024-12-05T11:16:14,912 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807f498965664cc59eb371576df91d53, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397370185 2024-12-05T11:16:14,913 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 392f9a942215472687e4b354002712fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733397371381 2024-12-05T11:16:14,913 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e26de87f74854c93877caf6825222b7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733397371381 2024-12-05T11:16:14,913 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bbc91b7bcd241d09bfdba749a71bf84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733397373623 2024-12-05T11:16:14,913 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e5adf8a28b4051b14c1a6f50aabda5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733397373623 2024-12-05T11:16:14,918 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:14,919 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e is 50, key is test_row_0/A:col10/1733397373631/Put/seqid=0 2024-12-05T11:16:14,919 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:14,919 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/d6edb01cda8548ef84da721f7e9c6325 is 50, key is test_row_0/B:col10/1733397373631/Put/seqid=0 2024-12-05T11:16:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742262_1438 (size=12493) 2024-12-05T11:16:14,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742263_1439 (size=12493) 2024-12-05T11:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:15,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T11:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:15,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6488eb937a114ad688f3e83aceb81d06 is 50, key is test_row_0/A:col10/1733397373802/Put/seqid=0 2024-12-05T11:16:15,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742264_1440 (size=14541) 2024-12-05T11:16:15,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397435061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:15,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397435168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:15,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,329 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e 2024-12-05T11:16:15,332 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/d6edb01cda8548ef84da721f7e9c6325 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/d6edb01cda8548ef84da721f7e9c6325 2024-12-05T11:16:15,332 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into dccf7a75fd0a45ab8cedc7c40a3d2b9e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:15,332 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:15,332 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=13, startTime=1733397374911; duration=0sec 2024-12-05T11:16:15,333 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:15,333 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:15,333 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:15,333 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:15,333 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:15,333 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,333 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/83715bc8bbc74b1a9277ac32e5ea96e2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9c4a2f85fcac4d9598e7e5ee568bafa9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0985395cc83d4169a8601900cee891c2] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=35.8 K 2024-12-05T11:16:15,334 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83715bc8bbc74b1a9277ac32e5ea96e2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733397370185 2024-12-05T11:16:15,334 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4a2f85fcac4d9598e7e5ee568bafa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733397371381 2024-12-05T11:16:15,334 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0985395cc83d4169a8601900cee891c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733397373623 2024-12-05T11:16:15,335 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into d6edb01cda8548ef84da721f7e9c6325(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:15,335 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:15,336 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=13, startTime=1733397374911; duration=0sec 2024-12-05T11:16:15,336 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:15,336 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:15,339 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:15,339 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9b4088d6edfd4e03be25cddfe4e76a77 is 50, key is test_row_0/C:col10/1733397373631/Put/seqid=0 2024-12-05T11:16:15,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742265_1441 (size=12493) 2024-12-05T11:16:15,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397435378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:15,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6488eb937a114ad688f3e83aceb81d06 2024-12-05T11:16:15,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3c1fd370d7ea4dd3a4947b4e17dbd84a is 50, key is test_row_0/B:col10/1733397373802/Put/seqid=0 2024-12-05T11:16:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742266_1442 (size=12151) 2024-12-05T11:16:15,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3c1fd370d7ea4dd3a4947b4e17dbd84a 2024-12-05T11:16:15,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f2bb8d61d723444787576411aa6821f9 is 50, key is test_row_0/C:col10/1733397373802/Put/seqid=0 2024-12-05T11:16:15,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742267_1443 (size=12151) 2024-12-05T11:16:15,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397435689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:15,753 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9b4088d6edfd4e03be25cddfe4e76a77 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9b4088d6edfd4e03be25cddfe4e76a77 2024-12-05T11:16:15,767 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into 9b4088d6edfd4e03be25cddfe4e76a77(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:15,767 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:15,767 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=13, startTime=1733397374911; duration=0sec 2024-12-05T11:16:15,767 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:15,767 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:15,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:15,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f2bb8d61d723444787576411aa6821f9 2024-12-05T11:16:15,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6488eb937a114ad688f3e83aceb81d06 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6488eb937a114ad688f3e83aceb81d06 2024-12-05T11:16:15,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6488eb937a114ad688f3e83aceb81d06, entries=200, sequenceid=192, filesize=14.2 K 2024-12-05T11:16:15,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3c1fd370d7ea4dd3a4947b4e17dbd84a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c1fd370d7ea4dd3a4947b4e17dbd84a 2024-12-05T11:16:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c1fd370d7ea4dd3a4947b4e17dbd84a, entries=150, sequenceid=192, filesize=11.9 K 2024-12-05T11:16:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f2bb8d61d723444787576411aa6821f9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f2bb8d61d723444787576411aa6821f9 2024-12-05T11:16:15,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f2bb8d61d723444787576411aa6821f9, entries=150, sequenceid=192, filesize=11.9 K 2024-12-05T11:16:15,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for e19ca153de371b50b73536c04e3fa097 in 842ms, sequenceid=192, compaction requested=false 2024-12-05T11:16:15,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:15,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T11:16:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:15,975 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T11:16:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2dd1083c788c4bd6aae4f0a97a344dda is 50, key is test_row_1/A:col10/1733397375060/Put/seqid=0 2024-12-05T11:16:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742268_1444 (size=9757) 2024-12-05T11:16:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397376273, max timestamp=1733397376300Num uniques 859; 2024-12-05T11:16:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397376273, max timestamp=1733397376300Num uniques 859; 2024-12-05T11:16:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397376273, max timestamp=1733397376300Num uniques 859; 2024-12-05T11:16:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:16,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:16,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397376357, max timestamp=1733397376375Num uniques 1018; 2024-12-05T11:16:16,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397376357, max timestamp=1733397376375Num uniques 1018; 2024-12-05T11:16:16,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:16:16,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:16:16,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733397376357, max timestamp=1733397376375Num uniques 1018; 2024-12-05T11:16:16,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T11:16:16,397 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2dd1083c788c4bd6aae4f0a97a344dda 2024-12-05T11:16:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/2c688a106e3b44518ca0abad8c901ad5 is 50, key is test_row_1/B:col10/1733397375060/Put/seqid=0 2024-12-05T11:16:16,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742269_1445 (size=9757) 2024-12-05T11:16:16,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733397376439, max timestamp=1733397376517Num uniques 1027; 2024-12-05T11:16:16,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-05T11:16:16,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733397376439, max timestamp=1733397376517Num uniques 1027; 2024-12-05T11:16:16,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-05T11:16:16,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733397376439, max timestamp=1733397376517Num uniques 1027; 2024-12-05T11:16:16,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-12-05T11:16:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733397376609, max timestamp=1733397376639Num uniques 1036; 2024-12-05T11:16:16,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-05T11:16:16,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733397376609, max timestamp=1733397376639Num uniques 1036; 2024-12-05T11:16:16,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-05T11:16:16,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733397376609, max timestamp=1733397376639Num uniques 1009; 2024-12-05T11:16:16,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13096, suffix heap size=132904,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=286 2024-12-05T11:16:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397436683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397436684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397436693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397436694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397436695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397436798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397436798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,810 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/2c688a106e3b44518ca0abad8c901ad5 2024-12-05T11:16:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397436809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397436809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397436809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:16,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/cc3cf9324d3f467d9598579580d90dc8 is 50, key is test_row_1/C:col10/1733397375060/Put/seqid=0 2024-12-05T11:16:16,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742270_1446 (size=9757) 2024-12-05T11:16:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397437008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397437008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397437019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397437019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397437019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,220 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/cc3cf9324d3f467d9598579580d90dc8 2024-12-05T11:16:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2dd1083c788c4bd6aae4f0a97a344dda as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2dd1083c788c4bd6aae4f0a97a344dda 2024-12-05T11:16:17,227 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2dd1083c788c4bd6aae4f0a97a344dda, entries=100, sequenceid=204, filesize=9.5 K 2024-12-05T11:16:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/2c688a106e3b44518ca0abad8c901ad5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/2c688a106e3b44518ca0abad8c901ad5 2024-12-05T11:16:17,230 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/2c688a106e3b44518ca0abad8c901ad5, entries=100, sequenceid=204, filesize=9.5 K 2024-12-05T11:16:17,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/cc3cf9324d3f467d9598579580d90dc8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/cc3cf9324d3f467d9598579580d90dc8 2024-12-05T11:16:17,234 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/cc3cf9324d3f467d9598579580d90dc8, entries=100, sequenceid=204, filesize=9.5 K 2024-12-05T11:16:17,235 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for e19ca153de371b50b73536c04e3fa097 in 1260ms, sequenceid=204, compaction requested=true 2024-12-05T11:16:17,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:17,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:17,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-05T11:16:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-05T11:16:17,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-05T11:16:17,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5650 sec 2024-12-05T11:16:17,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.5690 sec 2024-12-05T11:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:17,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=147.60 KB heapSize=412.92 KB 2024-12-05T11:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:17,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/d02d75c9dacd48c8966dca98cfae270c is 50, key is test_row_0/A:col10/1733397377320/Put/seqid=0 2024-12-05T11:16:17,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742271_1447 (size=14741) 2024-12-05T11:16:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397437335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397437338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397437339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397437339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397437341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397437449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397437449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397437658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397437659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/d02d75c9dacd48c8966dca98cfae270c 2024-12-05T11:16:17,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/0bc322f0a1024c6b92fc7f4c7d221e7d is 50, key is test_row_0/B:col10/1733397377320/Put/seqid=0 2024-12-05T11:16:17,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742272_1448 (size=12301) 2024-12-05T11:16:17,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/0bc322f0a1024c6b92fc7f4c7d221e7d 2024-12-05T11:16:17,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1ab4dde66cf044759d8014887e73657b is 50, key is test_row_0/C:col10/1733397377320/Put/seqid=0 2024-12-05T11:16:17,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742273_1449 (size=12301) 2024-12-05T11:16:17,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1ab4dde66cf044759d8014887e73657b 2024-12-05T11:16:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/d02d75c9dacd48c8966dca98cfae270c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/d02d75c9dacd48c8966dca98cfae270c 2024-12-05T11:16:17,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/d02d75c9dacd48c8966dca98cfae270c, entries=200, sequenceid=292, filesize=14.4 K 2024-12-05T11:16:17,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/0bc322f0a1024c6b92fc7f4c7d221e7d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0bc322f0a1024c6b92fc7f4c7d221e7d 2024-12-05T11:16:17,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0bc322f0a1024c6b92fc7f4c7d221e7d, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:16:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1ab4dde66cf044759d8014887e73657b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1ab4dde66cf044759d8014887e73657b 2024-12-05T11:16:17,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1ab4dde66cf044759d8014887e73657b, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:16:17,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.88 KB/422784, currentSize=33.54 KB/34350 for e19ca153de371b50b73536c04e3fa097 in 452ms, sequenceid=292, compaction requested=true 2024-12-05T11:16:17,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:17,774 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T11:16:17,774 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:17,775 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:17,775 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:17,775 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:17,775 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:17,775 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6488eb937a114ad688f3e83aceb81d06, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2dd1083c788c4bd6aae4f0a97a344dda, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/d02d75c9dacd48c8966dca98cfae270c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=50.3 K 2024-12-05T11:16:17,775 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:17,775 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:17,775 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/d6edb01cda8548ef84da721f7e9c6325, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c1fd370d7ea4dd3a4947b4e17dbd84a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/2c688a106e3b44518ca0abad8c901ad5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0bc322f0a1024c6b92fc7f4c7d221e7d] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=45.6 K 2024-12-05T11:16:17,775 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting dccf7a75fd0a45ab8cedc7c40a3d2b9e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733397373623 2024-12-05T11:16:17,775 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d6edb01cda8548ef84da721f7e9c6325, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733397373623 2024-12-05T11:16:17,776 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6488eb937a114ad688f3e83aceb81d06, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733397373802 2024-12-05T11:16:17,776 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c1fd370d7ea4dd3a4947b4e17dbd84a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733397373802 2024-12-05T11:16:17,776 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dd1083c788c4bd6aae4f0a97a344dda, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733397375026 2024-12-05T11:16:17,776 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c688a106e3b44518ca0abad8c901ad5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733397375026 2024-12-05T11:16:17,776 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc322f0a1024c6b92fc7f4c7d221e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397376680 2024-12-05T11:16:17,776 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02d75c9dacd48c8966dca98cfae270c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397376680 2024-12-05T11:16:17,785 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:17,785 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:17,785 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6d819f62c7174ff68cc32d7c87ff31b5 is 50, key is test_row_0/A:col10/1733397377320/Put/seqid=0 2024-12-05T11:16:17,785 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3c2336f55d39451c8500fa30eb8c1cb8 is 50, key is test_row_0/B:col10/1733397377320/Put/seqid=0 2024-12-05T11:16:17,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742275_1451 (size=12779) 2024-12-05T11:16:17,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742274_1450 (size=12779) 2024-12-05T11:16:17,804 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6d819f62c7174ff68cc32d7c87ff31b5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6d819f62c7174ff68cc32d7c87ff31b5 2024-12-05T11:16:17,808 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into 6d819f62c7174ff68cc32d7c87ff31b5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:17,808 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:17,808 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=12, startTime=1733397377773; duration=0sec 2024-12-05T11:16:17,808 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:17,808 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:17,808 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:17,810 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:17,810 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:17,810 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:17,810 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9b4088d6edfd4e03be25cddfe4e76a77, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f2bb8d61d723444787576411aa6821f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/cc3cf9324d3f467d9598579580d90dc8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1ab4dde66cf044759d8014887e73657b] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=45.6 K 2024-12-05T11:16:17,810 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4088d6edfd4e03be25cddfe4e76a77, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733397373623 2024-12-05T11:16:17,811 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2bb8d61d723444787576411aa6821f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733397373802 2024-12-05T11:16:17,811 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc3cf9324d3f467d9598579580d90dc8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733397375026 2024-12-05T11:16:17,811 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ab4dde66cf044759d8014887e73657b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397376680 2024-12-05T11:16:17,820 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:17,821 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f0aa5e0de1824ae5bb10fee37ebcf9c2 is 50, key is test_row_0/C:col10/1733397377320/Put/seqid=0 2024-12-05T11:16:17,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742276_1452 (size=12779) 2024-12-05T11:16:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T11:16:17,828 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:17,828 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-05T11:16:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:16:17,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:17,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:17,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:17,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:16:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:17,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:17,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4a00f787eae546ff885be85cb58f2686 is 50, key is test_row_0/A:col10/1733397377858/Put/seqid=0 2024-12-05T11:16:17,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742277_1453 (size=14741) 2024-12-05T11:16:17,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733397377878, max timestamp=1733397377905Num uniques 859; 2024-12-05T11:16:17,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733397377878, max timestamp=1733397377905Num uniques 859; 2024-12-05T11:16:17,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T11:16:17,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T11:16:17,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733397377878, max timestamp=1733397377905Num uniques 859; 2024-12-05T11:16:17,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T11:16:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:16:17,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397437943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397437953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397437953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397437969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:17,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T11:16:17,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:17,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:17,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:17,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:17,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:17,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397437978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397438059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397438059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397438059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T11:16:18,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:18,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:18,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:18,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:16:18,194 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/3c2336f55d39451c8500fa30eb8c1cb8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c2336f55d39451c8500fa30eb8c1cb8 2024-12-05T11:16:18,198 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into 3c2336f55d39451c8500fa30eb8c1cb8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:18,198 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:18,198 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=12, startTime=1733397377774; duration=0sec 2024-12-05T11:16:18,198 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:18,198 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:18,229 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/f0aa5e0de1824ae5bb10fee37ebcf9c2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0aa5e0de1824ae5bb10fee37ebcf9c2 2024-12-05T11:16:18,233 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into f0aa5e0de1824ae5bb10fee37ebcf9c2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:18,233 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:18,233 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=12, startTime=1733397377774; duration=0sec 2024-12-05T11:16:18,233 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:18,233 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:18,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4a00f787eae546ff885be85cb58f2686 2024-12-05T11:16:18,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/0dd7a463b65f4c83a2bdbe60b00b9169 is 50, key is test_row_0/B:col10/1733397377858/Put/seqid=0 2024-12-05T11:16:18,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742278_1454 (size=12301) 2024-12-05T11:16:18,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/0dd7a463b65f4c83a2bdbe60b00b9169 2024-12-05T11:16:18,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397438279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397438279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397438279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T11:16:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:18,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:18,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/d9c87a555daf43bbae9a108568e8f7db is 50, key is test_row_0/C:col10/1733397377858/Put/seqid=0 2024-12-05T11:16:18,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742279_1455 (size=12301) 2024-12-05T11:16:18,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/d9c87a555daf43bbae9a108568e8f7db 2024-12-05T11:16:18,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4a00f787eae546ff885be85cb58f2686 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4a00f787eae546ff885be85cb58f2686 2024-12-05T11:16:18,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4a00f787eae546ff885be85cb58f2686, entries=200, sequenceid=303, filesize=14.4 K 2024-12-05T11:16:18,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/0dd7a463b65f4c83a2bdbe60b00b9169 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0dd7a463b65f4c83a2bdbe60b00b9169 2024-12-05T11:16:18,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0dd7a463b65f4c83a2bdbe60b00b9169, entries=150, sequenceid=303, filesize=12.0 K 2024-12-05T11:16:18,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/d9c87a555daf43bbae9a108568e8f7db as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/d9c87a555daf43bbae9a108568e8f7db 2024-12-05T11:16:18,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/d9c87a555daf43bbae9a108568e8f7db, entries=150, sequenceid=303, filesize=12.0 K 2024-12-05T11:16:18,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e19ca153de371b50b73536c04e3fa097 in 475ms, sequenceid=303, compaction requested=false 2024-12-05T11:16:18,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:18,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T11:16:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:18,441 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-05T11:16:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:18,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2e9e18e644204a83a74caeb5a97bd3ab is 50, key is test_row_0/A:col10/1733397377951/Put/seqid=0 2024-12-05T11:16:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742280_1456 (size=12301) 2024-12-05T11:16:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:16:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:18,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397438509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397438515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397438588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397438589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397438589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397438628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397438628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397438838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397438838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:18,853 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2e9e18e644204a83a74caeb5a97bd3ab 2024-12-05T11:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/5c84ba273dd94fb5977c4a0dfac26c92 is 50, key is test_row_0/B:col10/1733397377951/Put/seqid=0 2024-12-05T11:16:18,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742281_1457 (size=12301) 2024-12-05T11:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:16:19,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397439108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397439108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397439108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397439148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397439158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,263 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/5c84ba273dd94fb5977c4a0dfac26c92 2024-12-05T11:16:19,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/ea0abee2aebd49aa8e4b01b04f616872 is 50, key is test_row_0/C:col10/1733397377951/Put/seqid=0 2024-12-05T11:16:19,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742282_1458 (size=12301) 2024-12-05T11:16:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397439658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397439668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:19,672 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/ea0abee2aebd49aa8e4b01b04f616872 2024-12-05T11:16:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2e9e18e644204a83a74caeb5a97bd3ab as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2e9e18e644204a83a74caeb5a97bd3ab 2024-12-05T11:16:19,678 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2e9e18e644204a83a74caeb5a97bd3ab, entries=150, sequenceid=340, filesize=12.0 K 2024-12-05T11:16:19,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/5c84ba273dd94fb5977c4a0dfac26c92 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/5c84ba273dd94fb5977c4a0dfac26c92 2024-12-05T11:16:19,681 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/5c84ba273dd94fb5977c4a0dfac26c92, entries=150, sequenceid=340, filesize=12.0 K 2024-12-05T11:16:19,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/ea0abee2aebd49aa8e4b01b04f616872 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ea0abee2aebd49aa8e4b01b04f616872 2024-12-05T11:16:19,685 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ea0abee2aebd49aa8e4b01b04f616872, entries=150, sequenceid=340, filesize=12.0 K 2024-12-05T11:16:19,685 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e19ca153de371b50b73536c04e3fa097 in 1245ms, sequenceid=340, compaction requested=true 2024-12-05T11:16:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-05T11:16:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-05T11:16:19,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T11:16:19,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-12-05T11:16:19,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8610 sec 2024-12-05T11:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T11:16:19,978 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:19,979 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-05T11:16:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:16:19,980 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:19,981 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:19,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:16:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:20,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T11:16:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:20,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/5adc2928a79e4235b08e04526d00e60a is 50, key is test_row_0/A:col10/1733397378514/Put/seqid=0 2024-12-05T11:16:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742283_1459 (size=17181) 2024-12-05T11:16:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397440181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397440182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397440192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:16:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397440299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397440299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397440308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397440508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397440508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397440519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/5adc2928a79e4235b08e04526d00e60a 2024-12-05T11:16:20,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/03d802d31a6a4cbeac04f033a608d4df is 50, key is test_row_0/B:col10/1733397378514/Put/seqid=0 2024-12-05T11:16:20,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742284_1460 (size=12301) 2024-12-05T11:16:20,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/03d802d31a6a4cbeac04f033a608d4df 2024-12-05T11:16:20,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/ade9c2ac8a74442086dff39a58c3a847 is 50, key is test_row_0/C:col10/1733397378514/Put/seqid=0 2024-12-05T11:16:20,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742285_1461 (size=12301) 2024-12-05T11:16:20,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:20,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:20,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:16:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397440678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397440678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,684 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,684 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397440818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397440818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397440828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:20,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:20,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/ade9c2ac8a74442086dff39a58c3a847 2024-12-05T11:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/5adc2928a79e4235b08e04526d00e60a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/5adc2928a79e4235b08e04526d00e60a 2024-12-05T11:16:20,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/5adc2928a79e4235b08e04526d00e60a, entries=250, sequenceid=352, filesize=16.8 K 2024-12-05T11:16:20,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/03d802d31a6a4cbeac04f033a608d4df as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/03d802d31a6a4cbeac04f033a608d4df 2024-12-05T11:16:20,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/03d802d31a6a4cbeac04f033a608d4df, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T11:16:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/ade9c2ac8a74442086dff39a58c3a847 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ade9c2ac8a74442086dff39a58c3a847 2024-12-05T11:16:20,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ade9c2ac8a74442086dff39a58c3a847, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T11:16:20,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e19ca153de371b50b73536c04e3fa097 in 851ms, sequenceid=352, compaction requested=true 2024-12-05T11:16:20,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:20,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:20,983 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:20,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:20,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:20,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:20,984 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:20,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:20,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:20,984 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57002 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:20,984 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:20,984 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:20,984 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:20,984 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,984 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:20,984 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c2336f55d39451c8500fa30eb8c1cb8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0dd7a463b65f4c83a2bdbe60b00b9169, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/5c84ba273dd94fb5977c4a0dfac26c92, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/03d802d31a6a4cbeac04f033a608d4df] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=48.5 K 2024-12-05T11:16:20,984 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6d819f62c7174ff68cc32d7c87ff31b5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4a00f787eae546ff885be85cb58f2686, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2e9e18e644204a83a74caeb5a97bd3ab, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/5adc2928a79e4235b08e04526d00e60a] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=55.7 K 2024-12-05T11:16:20,985 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c2336f55d39451c8500fa30eb8c1cb8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397376680 2024-12-05T11:16:20,985 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d819f62c7174ff68cc32d7c87ff31b5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397376680 2024-12-05T11:16:20,985 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd7a463b65f4c83a2bdbe60b00b9169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733397377322 2024-12-05T11:16:20,985 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a00f787eae546ff885be85cb58f2686, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733397377322 2024-12-05T11:16:20,986 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e9e18e644204a83a74caeb5a97bd3ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733397377922 2024-12-05T11:16:20,986 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c84ba273dd94fb5977c4a0dfac26c92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733397377922 2024-12-05T11:16:20,986 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5adc2928a79e4235b08e04526d00e60a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733397378497 2024-12-05T11:16:20,986 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d802d31a6a4cbeac04f033a608d4df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733397378514 2024-12-05T11:16:20,993 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:20,993 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:20,993 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/b7f0521d916e4c50988e2e9acafeb95f is 50, key is test_row_0/B:col10/1733397378514/Put/seqid=0 2024-12-05T11:16:20,994 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/23eed624b36644cab6e8798cdbaac617 is 50, key is test_row_0/A:col10/1733397378514/Put/seqid=0 2024-12-05T11:16:21,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742287_1463 (size=12915) 2024-12-05T11:16:21,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742286_1462 (size=12915) 2024-12-05T11:16:21,018 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/23eed624b36644cab6e8798cdbaac617 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/23eed624b36644cab6e8798cdbaac617 2024-12-05T11:16:21,021 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into 23eed624b36644cab6e8798cdbaac617(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:21,021 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:21,022 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=12, startTime=1733397380983; duration=0sec 2024-12-05T11:16:21,022 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:21,022 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:21,022 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T11:16:21,023 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T11:16:21,023 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:21,023 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:21,023 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0aa5e0de1824ae5bb10fee37ebcf9c2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/d9c87a555daf43bbae9a108568e8f7db, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ea0abee2aebd49aa8e4b01b04f616872, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ade9c2ac8a74442086dff39a58c3a847] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=48.5 K 2024-12-05T11:16:21,023 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0aa5e0de1824ae5bb10fee37ebcf9c2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397376680 2024-12-05T11:16:21,023 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9c87a555daf43bbae9a108568e8f7db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733397377322 2024-12-05T11:16:21,024 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0abee2aebd49aa8e4b01b04f616872, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733397377922 2024-12-05T11:16:21,024 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade9c2ac8a74442086dff39a58c3a847, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733397378514 2024-12-05T11:16:21,031 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:21,032 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/a459be384cbd4eaf8e73b3cf45f3f765 is 50, key is test_row_0/C:col10/1733397378514/Put/seqid=0 2024-12-05T11:16:21,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742288_1464 (size=12915) 2024-12-05T11:16:21,043 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/a459be384cbd4eaf8e73b3cf45f3f765 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a459be384cbd4eaf8e73b3cf45f3f765 2024-12-05T11:16:21,047 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into a459be384cbd4eaf8e73b3cf45f3f765(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:21,047 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:21,047 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=12, startTime=1733397380984; duration=0sec 2024-12-05T11:16:21,047 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:21,047 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:21,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:21,050 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:21,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/3dde2afcf5694bcc83b56d9e08a56fd5 is 50, key is test_row_0/A:col10/1733397380190/Put/seqid=0 2024-12-05T11:16:21,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742289_1465 (size=12301) 2024-12-05T11:16:21,056 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/3dde2afcf5694bcc83b56d9e08a56fd5 2024-12-05T11:16:21,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/f6eadc4df8fc48e6b9ff0993b7048fa3 is 50, key is test_row_0/B:col10/1733397380190/Put/seqid=0 2024-12-05T11:16:21,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742290_1466 (size=12301) 2024-12-05T11:16:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:16:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397441347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397441348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397441354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,409 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/b7f0521d916e4c50988e2e9acafeb95f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/b7f0521d916e4c50988e2e9acafeb95f 2024-12-05T11:16:21,413 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into b7f0521d916e4c50988e2e9acafeb95f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:21,413 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:21,413 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=12, startTime=1733397380984; duration=0sec 2024-12-05T11:16:21,413 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:21,413 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397441458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,468 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/f6eadc4df8fc48e6b9ff0993b7048fa3 2024-12-05T11:16:21,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/30130b468d5041f48c75403a64505da5 is 50, key is test_row_0/C:col10/1733397380190/Put/seqid=0 2024-12-05T11:16:21,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397441469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397441469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742291_1467 (size=12301) 2024-12-05T11:16:21,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397441668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397441678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397441679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:21,879 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/30130b468d5041f48c75403a64505da5 2024-12-05T11:16:21,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/3dde2afcf5694bcc83b56d9e08a56fd5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/3dde2afcf5694bcc83b56d9e08a56fd5 2024-12-05T11:16:21,886 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/3dde2afcf5694bcc83b56d9e08a56fd5, entries=150, sequenceid=374, filesize=12.0 K 2024-12-05T11:16:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/f6eadc4df8fc48e6b9ff0993b7048fa3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f6eadc4df8fc48e6b9ff0993b7048fa3 2024-12-05T11:16:21,889 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f6eadc4df8fc48e6b9ff0993b7048fa3, entries=150, sequenceid=374, filesize=12.0 K 2024-12-05T11:16:21,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/30130b468d5041f48c75403a64505da5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/30130b468d5041f48c75403a64505da5 2024-12-05T11:16:21,892 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/30130b468d5041f48c75403a64505da5, entries=150, sequenceid=374, filesize=12.0 K 2024-12-05T11:16:21,893 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e19ca153de371b50b73536c04e3fa097 in 842ms, sequenceid=374, compaction requested=false 2024-12-05T11:16:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T11:16:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-05T11:16:21,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T11:16:21,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-12-05T11:16:21,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9170 sec 2024-12-05T11:16:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:21,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:21,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:21,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/ec4ab0ae8fe5464d82f1fca0339b6129 is 50, key is test_row_0/A:col10/1733397381979/Put/seqid=0 2024-12-05T11:16:21,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742292_1468 (size=14741) 2024-12-05T11:16:22,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397442018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397442019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397442022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T11:16:22,129 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:22,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-05T11:16:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:22,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:22,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:22,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397442128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397442129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397442138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:22,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:22,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:22,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397442338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397442338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397442349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/ec4ab0ae8fe5464d82f1fca0339b6129 2024-12-05T11:16:22,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/bb977d82a1464c2d9f99495e006cc418 is 50, key is test_row_0/B:col10/1733397381979/Put/seqid=0 2024-12-05T11:16:22,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742293_1469 (size=12301) 2024-12-05T11:16:22,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:22,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:22,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397442648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397442648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397442658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42392 deadline: 1733397442698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,704 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42420 deadline: 1733397442709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:22,714 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in e19ca153de371b50b73536c04e3fa097 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:22,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:22,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:22,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:22,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/bb977d82a1464c2d9f99495e006cc418 2024-12-05T11:16:22,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1c93a111c345430ca17e1b7c53a359f8 is 50, key is test_row_0/C:col10/1733397381979/Put/seqid=0 2024-12-05T11:16:22,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742294_1470 (size=12301) 2024-12-05T11:16:22,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:22,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397443158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:23,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397443158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:23,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397443178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:23,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:23,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1c93a111c345430ca17e1b7c53a359f8 2024-12-05T11:16:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/ec4ab0ae8fe5464d82f1fca0339b6129 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/ec4ab0ae8fe5464d82f1fca0339b6129 2024-12-05T11:16:23,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/ec4ab0ae8fe5464d82f1fca0339b6129, entries=200, sequenceid=388, filesize=14.4 K 2024-12-05T11:16:23,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/bb977d82a1464c2d9f99495e006cc418 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/bb977d82a1464c2d9f99495e006cc418 2024-12-05T11:16:23,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/bb977d82a1464c2d9f99495e006cc418, entries=150, sequenceid=388, filesize=12.0 K 2024-12-05T11:16:23,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1c93a111c345430ca17e1b7c53a359f8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1c93a111c345430ca17e1b7c53a359f8 2024-12-05T11:16:23,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1c93a111c345430ca17e1b7c53a359f8, entries=150, sequenceid=388, filesize=12.0 K 2024-12-05T11:16:23,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e19ca153de371b50b73536c04e3fa097 in 1238ms, sequenceid=388, compaction requested=true 2024-12-05T11:16:23,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:23,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:23,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:23,218 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:23,218 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:23,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:23,220 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,220 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,220 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/b7f0521d916e4c50988e2e9acafeb95f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f6eadc4df8fc48e6b9ff0993b7048fa3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/bb977d82a1464c2d9f99495e006cc418] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=36.6 K 2024-12-05T11:16:23,220 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/23eed624b36644cab6e8798cdbaac617, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/3dde2afcf5694bcc83b56d9e08a56fd5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/ec4ab0ae8fe5464d82f1fca0339b6129] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=39.0 K 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23eed624b36644cab6e8798cdbaac617, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733397378514 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b7f0521d916e4c50988e2e9acafeb95f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733397378514 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dde2afcf5694bcc83b56d9e08a56fd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733397380165 2024-12-05T11:16:23,220 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f6eadc4df8fc48e6b9ff0993b7048fa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733397380165 2024-12-05T11:16:23,221 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting bb977d82a1464c2d9f99495e006cc418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733397381338 2024-12-05T11:16:23,221 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4ab0ae8fe5464d82f1fca0339b6129, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733397381338 2024-12-05T11:16:23,226 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:23,227 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/a3d6a625c03e48538821464cd1e7ddad is 50, key is test_row_0/A:col10/1733397381979/Put/seqid=0 2024-12-05T11:16:23,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742295_1471 (size=13017) 2024-12-05T11:16:23,237 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:23,238 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/ff7f3970ee18498e9de778fc7bbbb328 is 50, key is test_row_0/B:col10/1733397381979/Put/seqid=0 2024-12-05T11:16:23,245 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/a3d6a625c03e48538821464cd1e7ddad as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/a3d6a625c03e48538821464cd1e7ddad 2024-12-05T11:16:23,250 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into a3d6a625c03e48538821464cd1e7ddad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:23,250 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:23,250 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=13, startTime=1733397383218; duration=0sec 2024-12-05T11:16:23,250 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:23,250 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:23,250 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:23,251 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:23,251 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:23,251 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,251 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a459be384cbd4eaf8e73b3cf45f3f765, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/30130b468d5041f48c75403a64505da5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1c93a111c345430ca17e1b7c53a359f8] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=36.6 K 2024-12-05T11:16:23,251 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting a459be384cbd4eaf8e73b3cf45f3f765, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733397378514 2024-12-05T11:16:23,252 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30130b468d5041f48c75403a64505da5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733397380165 2024-12-05T11:16:23,252 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c93a111c345430ca17e1b7c53a359f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733397381338 2024-12-05T11:16:23,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742296_1472 (size=13017) 2024-12-05T11:16:23,258 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:23,258 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1a24f6850bbc4b7b9f7174e47d1176cd is 50, key is test_row_0/C:col10/1733397381979/Put/seqid=0 2024-12-05T11:16:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742297_1473 (size=13017) 2024-12-05T11:16:23,275 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/1a24f6850bbc4b7b9f7174e47d1176cd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1a24f6850bbc4b7b9f7174e47d1176cd 2024-12-05T11:16:23,279 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into 1a24f6850bbc4b7b9f7174e47d1176cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:23,279 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:23,279 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=13, startTime=1733397383219; duration=0sec 2024-12-05T11:16:23,279 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:23,279 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:23,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T11:16:23,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:23,352 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:16:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:23,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:23,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2794325c84d04edfa036cec2f9faf155 is 50, key is test_row_0/A:col10/1733397381997/Put/seqid=0 2024-12-05T11:16:23,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742298_1474 (size=12301) 2024-12-05T11:16:23,656 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/ff7f3970ee18498e9de778fc7bbbb328 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/ff7f3970ee18498e9de778fc7bbbb328 2024-12-05T11:16:23,660 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into ff7f3970ee18498e9de778fc7bbbb328(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:23,660 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:23,660 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=13, startTime=1733397383218; duration=0sec 2024-12-05T11:16:23,660 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:23,660 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:23,760 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2794325c84d04edfa036cec2f9faf155 2024-12-05T11:16:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/db6c5b4a5eb64f64a49d1f56c8e92fcf is 50, key is test_row_0/B:col10/1733397381997/Put/seqid=0 2024-12-05T11:16:23,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742299_1475 (size=12301) 2024-12-05T11:16:24,169 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/db6c5b4a5eb64f64a49d1f56c8e92fcf 2024-12-05T11:16:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:24,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. as already flushing 2024-12-05T11:16:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/a6e88e398f124ebaa8f58865402ed161 is 50, key is test_row_0/C:col10/1733397381997/Put/seqid=0 2024-12-05T11:16:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742300_1476 (size=12301) 2024-12-05T11:16:24,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397444239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397444240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397444241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397444348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397444348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397444348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397444558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397444558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397444558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,626 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/a6e88e398f124ebaa8f58865402ed161 2024-12-05T11:16:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/2794325c84d04edfa036cec2f9faf155 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2794325c84d04edfa036cec2f9faf155 2024-12-05T11:16:24,632 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2794325c84d04edfa036cec2f9faf155, entries=150, sequenceid=409, filesize=12.0 K 2024-12-05T11:16:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/db6c5b4a5eb64f64a49d1f56c8e92fcf as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/db6c5b4a5eb64f64a49d1f56c8e92fcf 2024-12-05T11:16:24,635 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/db6c5b4a5eb64f64a49d1f56c8e92fcf, entries=150, sequenceid=409, filesize=12.0 K 2024-12-05T11:16:24,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/a6e88e398f124ebaa8f58865402ed161 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a6e88e398f124ebaa8f58865402ed161 2024-12-05T11:16:24,638 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a6e88e398f124ebaa8f58865402ed161, entries=150, sequenceid=409, filesize=12.0 K 2024-12-05T11:16:24,639 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e19ca153de371b50b73536c04e3fa097 in 1287ms, sequenceid=409, compaction requested=false 2024-12-05T11:16:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T11:16:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-05T11:16:24,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T11:16:24,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5090 sec 2024-12-05T11:16:24,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.5130 sec 2024-12-05T11:16:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:24,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:16:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:24,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:24,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/8941e00d288b4f3ba2586225fea963f8 is 50, key is test_row_0/A:col10/1733397384238/Put/seqid=0 2024-12-05T11:16:24,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742301_1477 (size=12301) 2024-12-05T11:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397444900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397444901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397444902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397445018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397445018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397445019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397445228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397445228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397445228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/8941e00d288b4f3ba2586225fea963f8 2024-12-05T11:16:25,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/efbca88fd3e64996b3e23c211e00d731 is 50, key is test_row_0/B:col10/1733397384238/Put/seqid=0 2024-12-05T11:16:25,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742302_1478 (size=12301) 2024-12-05T11:16:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397445539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397445539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:25,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397445539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:25,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:25,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/efbca88fd3e64996b3e23c211e00d731 2024-12-05T11:16:25,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/5cb024e793bd4376a02f0fd2b3aa435c is 50, key is test_row_0/C:col10/1733397384238/Put/seqid=0 2024-12-05T11:16:25,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742303_1479 (size=12301) 2024-12-05T11:16:26,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1733397446048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:26,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42456 deadline: 1733397446048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:26,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42464 deadline: 1733397446058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 , the old value is region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., hostname=6706a97ae65f,42931,1733397243639, seqNum=2 because the exception is null or not the one we care about 2024-12-05T11:16:26,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/5cb024e793bd4376a02f0fd2b3aa435c 2024-12-05T11:16:26,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/8941e00d288b4f3ba2586225fea963f8 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/8941e00d288b4f3ba2586225fea963f8 2024-12-05T11:16:26,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/8941e00d288b4f3ba2586225fea963f8, entries=150, sequenceid=425, filesize=12.0 K 2024-12-05T11:16:26,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/efbca88fd3e64996b3e23c211e00d731 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/efbca88fd3e64996b3e23c211e00d731 2024-12-05T11:16:26,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/efbca88fd3e64996b3e23c211e00d731, entries=150, sequenceid=425, filesize=12.0 K 2024-12-05T11:16:26,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/5cb024e793bd4376a02f0fd2b3aa435c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5cb024e793bd4376a02f0fd2b3aa435c 2024-12-05T11:16:26,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5cb024e793bd4376a02f0fd2b3aa435c, entries=150, sequenceid=425, filesize=12.0 K 2024-12-05T11:16:26,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e19ca153de371b50b73536c04e3fa097 in 1241ms, sequenceid=425, compaction requested=true 2024-12-05T11:16:26,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:26,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:26,115 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:26,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:26,115 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:26,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:26,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:26,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e19ca153de371b50b73536c04e3fa097:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:26,116 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:26,116 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:26,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:26,116 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/B is initiating minor compaction (all files) 2024-12-05T11:16:26,116 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/A is initiating minor compaction (all files) 2024-12-05T11:16:26,116 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/A in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:26,116 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/B in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:26,116 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/a3d6a625c03e48538821464cd1e7ddad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2794325c84d04edfa036cec2f9faf155, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/8941e00d288b4f3ba2586225fea963f8] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=36.7 K 2024-12-05T11:16:26,116 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/ff7f3970ee18498e9de778fc7bbbb328, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/db6c5b4a5eb64f64a49d1f56c8e92fcf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/efbca88fd3e64996b3e23c211e00d731] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=36.7 K 2024-12-05T11:16:26,116 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d6a625c03e48538821464cd1e7ddad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733397381338 2024-12-05T11:16:26,117 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7f3970ee18498e9de778fc7bbbb328, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733397381338 2024-12-05T11:16:26,117 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2794325c84d04edfa036cec2f9faf155, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733397381997 2024-12-05T11:16:26,117 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting db6c5b4a5eb64f64a49d1f56c8e92fcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733397381997 2024-12-05T11:16:26,117 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8941e00d288b4f3ba2586225fea963f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733397384238 2024-12-05T11:16:26,117 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting efbca88fd3e64996b3e23c211e00d731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733397384238 2024-12-05T11:16:26,125 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#A#compaction#408 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:26,125 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/31a126f61a0246afa94a6452bf6d04b4 is 50, key is test_row_0/A:col10/1733397384238/Put/seqid=0 2024-12-05T11:16:26,125 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:26,126 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/e0eb768fa03d45709fbf6036bd379169 is 50, key is test_row_0/B:col10/1733397384238/Put/seqid=0 2024-12-05T11:16:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742304_1480 (size=13119) 2024-12-05T11:16:26,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742305_1481 (size=13119) 2024-12-05T11:16:26,134 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/31a126f61a0246afa94a6452bf6d04b4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/31a126f61a0246afa94a6452bf6d04b4 2024-12-05T11:16:26,134 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/e0eb768fa03d45709fbf6036bd379169 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e0eb768fa03d45709fbf6036bd379169 2024-12-05T11:16:26,138 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/B of e19ca153de371b50b73536c04e3fa097 into e0eb768fa03d45709fbf6036bd379169(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:26,138 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/A of e19ca153de371b50b73536c04e3fa097 into 31a126f61a0246afa94a6452bf6d04b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:26,138 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/B, priority=13, startTime=1733397386115; duration=0sec 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:26,138 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/A, priority=13, startTime=1733397386115; duration=0sec 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:B 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:A 2024-12-05T11:16:26,138 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:26,139 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:26,139 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): e19ca153de371b50b73536c04e3fa097/C is initiating minor compaction (all files) 2024-12-05T11:16:26,139 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e19ca153de371b50b73536c04e3fa097/C in TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:26,139 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1a24f6850bbc4b7b9f7174e47d1176cd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a6e88e398f124ebaa8f58865402ed161, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5cb024e793bd4376a02f0fd2b3aa435c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp, totalSize=36.7 K 2024-12-05T11:16:26,140 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a24f6850bbc4b7b9f7174e47d1176cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733397381338 2024-12-05T11:16:26,140 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting a6e88e398f124ebaa8f58865402ed161, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733397381997 2024-12-05T11:16:26,141 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb024e793bd4376a02f0fd2b3aa435c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733397384238 2024-12-05T11:16:26,148 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e19ca153de371b50b73536c04e3fa097#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:26,149 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/29bdde8c169c4cfe806b67866ece94de is 50, key is test_row_0/C:col10/1733397384238/Put/seqid=0 2024-12-05T11:16:26,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742306_1482 (size=13119) 2024-12-05T11:16:26,224 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-12-05T11:16:26,224 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,224 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,225 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,225 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-12-05T11:16:26,225 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,225 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,225 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,227 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-12-05T11:16:26,227 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,227 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,227 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,228 INFO [Thread-1857 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1857. 2024-12-05T11:16:26,228 DEBUG [Thread-1857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,228 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,228 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,229 INFO [Thread-1855 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1855. 2024-12-05T11:16:26,229 DEBUG [Thread-1855 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,229 DEBUG [Thread-1855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,229 DEBUG [Thread-1855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T11:16:26,289 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:26,567 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/29bdde8c169c4cfe806b67866ece94de as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/29bdde8c169c4cfe806b67866ece94de 2024-12-05T11:16:26,570 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e19ca153de371b50b73536c04e3fa097/C of e19ca153de371b50b73536c04e3fa097 into 29bdde8c169c4cfe806b67866ece94de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:26,570 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:26,571 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., storeName=e19ca153de371b50b73536c04e3fa097/C, priority=13, startTime=1733397386116; duration=0sec 2024-12-05T11:16:26,571 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:26,571 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e19ca153de371b50b73536c04e3fa097:C 2024-12-05T11:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:26,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T11:16:26,722 INFO [Thread-1850 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1850. 2024-12-05T11:16:26,722 DEBUG [Thread-1850 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,723 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:26,723 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:26,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:26,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6bbec5cb6f624d72978f50fd45a955ea is 50, key is test_row_0/A:col10/1733397384890/Put/seqid=0 2024-12-05T11:16:26,730 INFO [Thread-1846 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1846. 2024-12-05T11:16:26,730 DEBUG [Thread-1846 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:26,730 DEBUG [Thread-1846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,730 DEBUG [Thread-1846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:26,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:26,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742307_1483 (size=12301) 2024-12-05T11:16:27,063 INFO [Thread-1852 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1852. 2024-12-05T11:16:27,063 DEBUG [Thread-1852 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:27,063 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,064 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:27,072 INFO [Thread-1844 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1844. 2024-12-05T11:16:27,072 INFO [Thread-1848 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1848. 2024-12-05T11:16:27,072 DEBUG [Thread-1844 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:27,072 DEBUG [Thread-1848 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:27,072 DEBUG [Thread-1844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,072 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,072 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,072 DEBUG [Thread-1844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:27,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:27,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:16:27,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-05T11:16:27,072 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 174 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1562 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4676 rows 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1552 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4656 rows 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1556 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4659 rows 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1553 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4658 rows 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1567 2024-12-05T11:16:27,073 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4700 rows 2024-12-05T11:16:27,074 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:16:27,074 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:16:27,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:27,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:27,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:16:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:27,078 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397387078"}]},"ts":"1733397387078"} 2024-12-05T11:16:27,079 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:16:27,079 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:16:27,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:16:27,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, UNASSIGN}] 2024-12-05T11:16:27,082 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, UNASSIGN 2024-12-05T11:16:27,083 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=e19ca153de371b50b73536c04e3fa097, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:27,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, UNASSIGN because future has completed 2024-12-05T11:16:27,084 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:16:27,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:16:27,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6bbec5cb6f624d72978f50fd45a955ea 2024-12-05T11:16:27,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/57302604b2f74ce28aa3de8105013030 is 50, key is test_row_0/B:col10/1733397384890/Put/seqid=0 2024-12-05T11:16:27,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742308_1484 (size=12301) 2024-12-05T11:16:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:27,237 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:27,237 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:16:27,237 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing e19ca153de371b50b73536c04e3fa097, disabling compactions & flushes 2024-12-05T11:16:27,237 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:27,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/57302604b2f74ce28aa3de8105013030 2024-12-05T11:16:27,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/2d682d2cd3844555943f264b74e67f4f is 50, key is test_row_0/C:col10/1733397384890/Put/seqid=0 2024-12-05T11:16:27,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742309_1485 (size=12301) 2024-12-05T11:16:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:27,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/2d682d2cd3844555943f264b74e67f4f 2024-12-05T11:16:27,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/6bbec5cb6f624d72978f50fd45a955ea as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6bbec5cb6f624d72978f50fd45a955ea 2024-12-05T11:16:27,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6bbec5cb6f624d72978f50fd45a955ea, entries=150, sequenceid=446, filesize=12.0 K 2024-12-05T11:16:27,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/57302604b2f74ce28aa3de8105013030 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/57302604b2f74ce28aa3de8105013030 2024-12-05T11:16:27,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/57302604b2f74ce28aa3de8105013030, entries=150, sequenceid=446, filesize=12.0 K 2024-12-05T11:16:27,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/2d682d2cd3844555943f264b74e67f4f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/2d682d2cd3844555943f264b74e67f4f 2024-12-05T11:16:27,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/2d682d2cd3844555943f264b74e67f4f, entries=150, sequenceid=446, filesize=12.0 K 2024-12-05T11:16:27,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for e19ca153de371b50b73536c04e3fa097 in 1269ms, sequenceid=446, compaction requested=false 2024-12-05T11:16:27,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e19ca153de371b50b73536c04e3fa097: 2024-12-05T11:16:27,991 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:27,991 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:27,991 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. after waiting 0 ms 2024-12-05T11:16:27,991 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:27,991 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing e19ca153de371b50b73536c04e3fa097 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T11:16:27,992 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=A 2024-12-05T11:16:27,992 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:27,992 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=B 2024-12-05T11:16:27,992 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:27,992 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e19ca153de371b50b73536c04e3fa097, store=C 2024-12-05T11:16:27,992 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:27,995 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4e33a18574fa499f80007c94bb0621e0 is 50, key is test_row_1/A:col10/1733397387069/Put/seqid=0 2024-12-05T11:16:27,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742310_1486 (size=9857) 2024-12-05T11:16:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:28,399 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4e33a18574fa499f80007c94bb0621e0 2024-12-05T11:16:28,405 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/6dbf3712fb7c4cbdb3567b5faf99bb4b is 50, key is test_row_1/B:col10/1733397387069/Put/seqid=0 2024-12-05T11:16:28,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742311_1487 (size=9857) 2024-12-05T11:16:28,816 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/6dbf3712fb7c4cbdb3567b5faf99bb4b 2024-12-05T11:16:28,822 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9bddb9e85dbb47b3b7b73f4e72d73027 is 50, key is test_row_1/C:col10/1733397387069/Put/seqid=0 2024-12-05T11:16:28,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742312_1488 (size=9857) 2024-12-05T11:16:29,226 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9bddb9e85dbb47b3b7b73f4e72d73027 2024-12-05T11:16:29,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/A/4e33a18574fa499f80007c94bb0621e0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4e33a18574fa499f80007c94bb0621e0 2024-12-05T11:16:29,232 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4e33a18574fa499f80007c94bb0621e0, entries=100, sequenceid=453, filesize=9.6 K 2024-12-05T11:16:29,233 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/B/6dbf3712fb7c4cbdb3567b5faf99bb4b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/6dbf3712fb7c4cbdb3567b5faf99bb4b 2024-12-05T11:16:29,235 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/6dbf3712fb7c4cbdb3567b5faf99bb4b, entries=100, sequenceid=453, filesize=9.6 K 2024-12-05T11:16:29,236 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/.tmp/C/9bddb9e85dbb47b3b7b73f4e72d73027 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9bddb9e85dbb47b3b7b73f4e72d73027 2024-12-05T11:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:29,239 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9bddb9e85dbb47b3b7b73f4e72d73027, entries=100, sequenceid=453, filesize=9.6 K 2024-12-05T11:16:29,240 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e19ca153de371b50b73536c04e3fa097 in 1249ms, sequenceid=453, compaction requested=true 2024-12-05T11:16:29,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4898c899c266471eb426470aa9ceab8a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/e414242a7eba4759a1896601350c3e6b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1cb0477d7af64747ba4f81d011cef9c9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/93f9ccad03304bbbae62e30afa861388, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/77ad59f392ed4f90b4902cc7c8e2f922, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1b32933ba0f24abd84272a7b1c8947d9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1d52b4d1b0b64145b9b758f022af3545, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/807f498965664cc59eb371576df91d53, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/392f9a942215472687e4b354002712fb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/07e5adf8a28b4051b14c1a6f50aabda5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6488eb937a114ad688f3e83aceb81d06, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2dd1083c788c4bd6aae4f0a97a344dda, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/d02d75c9dacd48c8966dca98cfae270c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6d819f62c7174ff68cc32d7c87ff31b5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4a00f787eae546ff885be85cb58f2686, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2e9e18e644204a83a74caeb5a97bd3ab, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/5adc2928a79e4235b08e04526d00e60a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/23eed624b36644cab6e8798cdbaac617, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/3dde2afcf5694bcc83b56d9e08a56fd5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/ec4ab0ae8fe5464d82f1fca0339b6129, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/a3d6a625c03e48538821464cd1e7ddad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2794325c84d04edfa036cec2f9faf155, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/8941e00d288b4f3ba2586225fea963f8] to archive 2024-12-05T11:16:29,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:16:29,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4898c899c266471eb426470aa9ceab8a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4898c899c266471eb426470aa9ceab8a 2024-12-05T11:16:29,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/e414242a7eba4759a1896601350c3e6b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/e414242a7eba4759a1896601350c3e6b 2024-12-05T11:16:29,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1cb0477d7af64747ba4f81d011cef9c9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1cb0477d7af64747ba4f81d011cef9c9 2024-12-05T11:16:29,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/93f9ccad03304bbbae62e30afa861388 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/93f9ccad03304bbbae62e30afa861388 2024-12-05T11:16:29,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/77ad59f392ed4f90b4902cc7c8e2f922 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/77ad59f392ed4f90b4902cc7c8e2f922 2024-12-05T11:16:29,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1b32933ba0f24abd84272a7b1c8947d9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1b32933ba0f24abd84272a7b1c8947d9 2024-12-05T11:16:29,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1d52b4d1b0b64145b9b758f022af3545 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/1d52b4d1b0b64145b9b758f022af3545 2024-12-05T11:16:29,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/807f498965664cc59eb371576df91d53 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/807f498965664cc59eb371576df91d53 2024-12-05T11:16:29,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/392f9a942215472687e4b354002712fb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/392f9a942215472687e4b354002712fb 2024-12-05T11:16:29,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/07e5adf8a28b4051b14c1a6f50aabda5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/07e5adf8a28b4051b14c1a6f50aabda5 2024-12-05T11:16:29,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/dccf7a75fd0a45ab8cedc7c40a3d2b9e 2024-12-05T11:16:29,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6488eb937a114ad688f3e83aceb81d06 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6488eb937a114ad688f3e83aceb81d06 2024-12-05T11:16:29,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2dd1083c788c4bd6aae4f0a97a344dda to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2dd1083c788c4bd6aae4f0a97a344dda 2024-12-05T11:16:29,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/d02d75c9dacd48c8966dca98cfae270c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/d02d75c9dacd48c8966dca98cfae270c 2024-12-05T11:16:29,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6d819f62c7174ff68cc32d7c87ff31b5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6d819f62c7174ff68cc32d7c87ff31b5 2024-12-05T11:16:29,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4a00f787eae546ff885be85cb58f2686 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4a00f787eae546ff885be85cb58f2686 2024-12-05T11:16:29,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2e9e18e644204a83a74caeb5a97bd3ab to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2e9e18e644204a83a74caeb5a97bd3ab 2024-12-05T11:16:29,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/5adc2928a79e4235b08e04526d00e60a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/5adc2928a79e4235b08e04526d00e60a 2024-12-05T11:16:29,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/23eed624b36644cab6e8798cdbaac617 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/23eed624b36644cab6e8798cdbaac617 2024-12-05T11:16:29,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/3dde2afcf5694bcc83b56d9e08a56fd5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/3dde2afcf5694bcc83b56d9e08a56fd5 2024-12-05T11:16:29,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/ec4ab0ae8fe5464d82f1fca0339b6129 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/ec4ab0ae8fe5464d82f1fca0339b6129 2024-12-05T11:16:29,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/a3d6a625c03e48538821464cd1e7ddad to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/a3d6a625c03e48538821464cd1e7ddad 2024-12-05T11:16:29,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2794325c84d04edfa036cec2f9faf155 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/2794325c84d04edfa036cec2f9faf155 2024-12-05T11:16:29,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/8941e00d288b4f3ba2586225fea963f8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/8941e00d288b4f3ba2586225fea963f8 2024-12-05T11:16:29,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1790b222c2ca401ca16699f103860e66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/96938823e50540eca52a8354df255506, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3b0c9b36f70d4c958af0818650372100, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/c42a058672504d3ea87b1b71a5597c7b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1697c1dbae9047dc9d1361e58b90200a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/442a0ed4cfdf44bc888a338f5f653b65, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/fae69641b42d492782740ad7906d562c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f9f671314bc44f3bae98b698922d8958, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e26de87f74854c93877caf6825222b7b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/d6edb01cda8548ef84da721f7e9c6325, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/9bbc91b7bcd241d09bfdba749a71bf84, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c1fd370d7ea4dd3a4947b4e17dbd84a, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/2c688a106e3b44518ca0abad8c901ad5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c2336f55d39451c8500fa30eb8c1cb8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0bc322f0a1024c6b92fc7f4c7d221e7d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0dd7a463b65f4c83a2bdbe60b00b9169, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/5c84ba273dd94fb5977c4a0dfac26c92, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/b7f0521d916e4c50988e2e9acafeb95f, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/03d802d31a6a4cbeac04f033a608d4df, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f6eadc4df8fc48e6b9ff0993b7048fa3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/ff7f3970ee18498e9de778fc7bbbb328, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/bb977d82a1464c2d9f99495e006cc418, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/db6c5b4a5eb64f64a49d1f56c8e92fcf, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/efbca88fd3e64996b3e23c211e00d731] to archive 2024-12-05T11:16:29,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:16:29,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1790b222c2ca401ca16699f103860e66 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1790b222c2ca401ca16699f103860e66 2024-12-05T11:16:29,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/96938823e50540eca52a8354df255506 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/96938823e50540eca52a8354df255506 2024-12-05T11:16:29,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3b0c9b36f70d4c958af0818650372100 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3b0c9b36f70d4c958af0818650372100 2024-12-05T11:16:29,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/c42a058672504d3ea87b1b71a5597c7b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/c42a058672504d3ea87b1b71a5597c7b 2024-12-05T11:16:29,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1697c1dbae9047dc9d1361e58b90200a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/1697c1dbae9047dc9d1361e58b90200a 2024-12-05T11:16:29,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/442a0ed4cfdf44bc888a338f5f653b65 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/442a0ed4cfdf44bc888a338f5f653b65 2024-12-05T11:16:29,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/fae69641b42d492782740ad7906d562c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/fae69641b42d492782740ad7906d562c 2024-12-05T11:16:29,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f9f671314bc44f3bae98b698922d8958 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f9f671314bc44f3bae98b698922d8958 2024-12-05T11:16:29,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e26de87f74854c93877caf6825222b7b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e26de87f74854c93877caf6825222b7b 2024-12-05T11:16:29,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/d6edb01cda8548ef84da721f7e9c6325 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/d6edb01cda8548ef84da721f7e9c6325 2024-12-05T11:16:29,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/9bbc91b7bcd241d09bfdba749a71bf84 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/9bbc91b7bcd241d09bfdba749a71bf84 2024-12-05T11:16:29,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c1fd370d7ea4dd3a4947b4e17dbd84a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c1fd370d7ea4dd3a4947b4e17dbd84a 2024-12-05T11:16:29,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/2c688a106e3b44518ca0abad8c901ad5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/2c688a106e3b44518ca0abad8c901ad5 2024-12-05T11:16:29,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c2336f55d39451c8500fa30eb8c1cb8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/3c2336f55d39451c8500fa30eb8c1cb8 2024-12-05T11:16:29,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0bc322f0a1024c6b92fc7f4c7d221e7d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0bc322f0a1024c6b92fc7f4c7d221e7d 2024-12-05T11:16:29,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0dd7a463b65f4c83a2bdbe60b00b9169 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/0dd7a463b65f4c83a2bdbe60b00b9169 2024-12-05T11:16:29,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/5c84ba273dd94fb5977c4a0dfac26c92 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/5c84ba273dd94fb5977c4a0dfac26c92 2024-12-05T11:16:29,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/b7f0521d916e4c50988e2e9acafeb95f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/b7f0521d916e4c50988e2e9acafeb95f 2024-12-05T11:16:29,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/03d802d31a6a4cbeac04f033a608d4df to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/03d802d31a6a4cbeac04f033a608d4df 2024-12-05T11:16:29,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f6eadc4df8fc48e6b9ff0993b7048fa3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/f6eadc4df8fc48e6b9ff0993b7048fa3 2024-12-05T11:16:29,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/ff7f3970ee18498e9de778fc7bbbb328 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/ff7f3970ee18498e9de778fc7bbbb328 2024-12-05T11:16:29,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/bb977d82a1464c2d9f99495e006cc418 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/bb977d82a1464c2d9f99495e006cc418 2024-12-05T11:16:29,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/db6c5b4a5eb64f64a49d1f56c8e92fcf to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/db6c5b4a5eb64f64a49d1f56c8e92fcf 2024-12-05T11:16:29,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/efbca88fd3e64996b3e23c211e00d731 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/efbca88fd3e64996b3e23c211e00d731 2024-12-05T11:16:29,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/76b862881dfc41c8a642994e5b730d22, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/c767340d9fdf40249ab6d614c3b1aaac, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/fbb27663cc0a46d5829f217296f32cb8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/6259bf3c9f424d5182b5b96a47e45ce1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5707aa27a74d4a99951c685347faed14, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0c8261dac1d4b4f84b56d68de214758, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/83715bc8bbc74b1a9277ac32e5ea96e2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0ff0654997e34cde8c88c300af71a10c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9c4a2f85fcac4d9598e7e5ee568bafa9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9b4088d6edfd4e03be25cddfe4e76a77, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0985395cc83d4169a8601900cee891c2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f2bb8d61d723444787576411aa6821f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/cc3cf9324d3f467d9598579580d90dc8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0aa5e0de1824ae5bb10fee37ebcf9c2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1ab4dde66cf044759d8014887e73657b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/d9c87a555daf43bbae9a108568e8f7db, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ea0abee2aebd49aa8e4b01b04f616872, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a459be384cbd4eaf8e73b3cf45f3f765, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ade9c2ac8a74442086dff39a58c3a847, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/30130b468d5041f48c75403a64505da5, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1a24f6850bbc4b7b9f7174e47d1176cd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1c93a111c345430ca17e1b7c53a359f8, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a6e88e398f124ebaa8f58865402ed161, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5cb024e793bd4376a02f0fd2b3aa435c] to archive 2024-12-05T11:16:29,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:16:29,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/76b862881dfc41c8a642994e5b730d22 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/76b862881dfc41c8a642994e5b730d22 2024-12-05T11:16:29,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/c767340d9fdf40249ab6d614c3b1aaac to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/c767340d9fdf40249ab6d614c3b1aaac 2024-12-05T11:16:29,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/fbb27663cc0a46d5829f217296f32cb8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/fbb27663cc0a46d5829f217296f32cb8 2024-12-05T11:16:29,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/6259bf3c9f424d5182b5b96a47e45ce1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/6259bf3c9f424d5182b5b96a47e45ce1 2024-12-05T11:16:29,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5707aa27a74d4a99951c685347faed14 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5707aa27a74d4a99951c685347faed14 2024-12-05T11:16:29,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0c8261dac1d4b4f84b56d68de214758 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0c8261dac1d4b4f84b56d68de214758 2024-12-05T11:16:29,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/83715bc8bbc74b1a9277ac32e5ea96e2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/83715bc8bbc74b1a9277ac32e5ea96e2 2024-12-05T11:16:29,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0ff0654997e34cde8c88c300af71a10c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0ff0654997e34cde8c88c300af71a10c 2024-12-05T11:16:29,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9c4a2f85fcac4d9598e7e5ee568bafa9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9c4a2f85fcac4d9598e7e5ee568bafa9 2024-12-05T11:16:29,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9b4088d6edfd4e03be25cddfe4e76a77 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9b4088d6edfd4e03be25cddfe4e76a77 2024-12-05T11:16:29,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0985395cc83d4169a8601900cee891c2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/0985395cc83d4169a8601900cee891c2 2024-12-05T11:16:29,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f2bb8d61d723444787576411aa6821f9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f2bb8d61d723444787576411aa6821f9 2024-12-05T11:16:29,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/cc3cf9324d3f467d9598579580d90dc8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/cc3cf9324d3f467d9598579580d90dc8 2024-12-05T11:16:29,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0aa5e0de1824ae5bb10fee37ebcf9c2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/f0aa5e0de1824ae5bb10fee37ebcf9c2 2024-12-05T11:16:29,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1ab4dde66cf044759d8014887e73657b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1ab4dde66cf044759d8014887e73657b 2024-12-05T11:16:29,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/d9c87a555daf43bbae9a108568e8f7db to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/d9c87a555daf43bbae9a108568e8f7db 2024-12-05T11:16:29,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ea0abee2aebd49aa8e4b01b04f616872 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ea0abee2aebd49aa8e4b01b04f616872 2024-12-05T11:16:29,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a459be384cbd4eaf8e73b3cf45f3f765 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a459be384cbd4eaf8e73b3cf45f3f765 2024-12-05T11:16:29,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ade9c2ac8a74442086dff39a58c3a847 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/ade9c2ac8a74442086dff39a58c3a847 2024-12-05T11:16:29,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/30130b468d5041f48c75403a64505da5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/30130b468d5041f48c75403a64505da5 2024-12-05T11:16:29,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1a24f6850bbc4b7b9f7174e47d1176cd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1a24f6850bbc4b7b9f7174e47d1176cd 2024-12-05T11:16:29,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1c93a111c345430ca17e1b7c53a359f8 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/1c93a111c345430ca17e1b7c53a359f8 2024-12-05T11:16:29,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a6e88e398f124ebaa8f58865402ed161 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/a6e88e398f124ebaa8f58865402ed161 2024-12-05T11:16:29,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5cb024e793bd4376a02f0fd2b3aa435c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/5cb024e793bd4376a02f0fd2b3aa435c 2024-12-05T11:16:29,317 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/recovered.edits/456.seqid, newMaxSeqId=456, maxSeqId=1 2024-12-05T11:16:29,318 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. 2024-12-05T11:16:29,318 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for e19ca153de371b50b73536c04e3fa097: Waiting for close lock at 1733397387237Running coprocessor pre-close hooks at 1733397387237Disabling compacts and flushes for region at 1733397387237Disabling writes for close at 1733397387991 (+754 ms)Obtaining lock to block concurrent updates at 1733397387992 (+1 ms)Preparing flush snapshotting stores in e19ca153de371b50b73536c04e3fa097 at 1733397387992Finished memstore snapshotting TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733397387992Flushing stores of TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097. at 1733397387992Flushing e19ca153de371b50b73536c04e3fa097/A: creating writer at 1733397387992Flushing e19ca153de371b50b73536c04e3fa097/A: appending metadata at 1733397387995 (+3 ms)Flushing e19ca153de371b50b73536c04e3fa097/A: closing flushed file at 1733397387995Flushing e19ca153de371b50b73536c04e3fa097/B: creating writer at 1733397388402 (+407 ms)Flushing e19ca153de371b50b73536c04e3fa097/B: appending metadata at 1733397388404 (+2 ms)Flushing e19ca153de371b50b73536c04e3fa097/B: closing flushed file at 1733397388404Flushing e19ca153de371b50b73536c04e3fa097/C: creating writer at 1733397388819 (+415 ms)Flushing e19ca153de371b50b73536c04e3fa097/C: appending metadata at 1733397388821 (+2 ms)Flushing e19ca153de371b50b73536c04e3fa097/C: closing flushed file at 1733397388821Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@127527cc: reopening flushed file at 1733397389229 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51469cde: reopening flushed file at 1733397389233 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a9f80c: reopening flushed file at 1733397389235 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e19ca153de371b50b73536c04e3fa097 in 1249ms, sequenceid=453, compaction requested=true at 1733397389240 (+5 ms)Writing region close event to WAL at 1733397389315 (+75 ms)Running coprocessor post-close hooks at 1733397389317 (+2 ms)Closed at 1733397389317 2024-12-05T11:16:29,319 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:29,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=e19ca153de371b50b73536c04e3fa097, regionState=CLOSED 2024-12-05T11:16:29,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:16:29,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T11:16:29,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure e19ca153de371b50b73536c04e3fa097, server=6706a97ae65f,42931,1733397243639 in 2.2380 sec 2024-12-05T11:16:29,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-12-05T11:16:29,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e19ca153de371b50b73536c04e3fa097, UNASSIGN in 2.2420 sec 2024-12-05T11:16:29,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-05T11:16:29,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2450 sec 2024-12-05T11:16:29,327 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397389327"}]},"ts":"1733397389327"} 2024-12-05T11:16:29,328 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:16:29,328 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:16:29,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2530 sec 2024-12-05T11:16:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T11:16:31,250 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:31,251 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:16:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,256 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:16:31,258 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,262 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:31,264 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/recovered.edits] 2024-12-05T11:16:31,268 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/31a126f61a0246afa94a6452bf6d04b4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/31a126f61a0246afa94a6452bf6d04b4 2024-12-05T11:16:31,270 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4e33a18574fa499f80007c94bb0621e0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/4e33a18574fa499f80007c94bb0621e0 2024-12-05T11:16:31,273 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6bbec5cb6f624d72978f50fd45a955ea to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/A/6bbec5cb6f624d72978f50fd45a955ea 2024-12-05T11:16:31,277 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/57302604b2f74ce28aa3de8105013030 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/57302604b2f74ce28aa3de8105013030 2024-12-05T11:16:31,279 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/6dbf3712fb7c4cbdb3567b5faf99bb4b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/6dbf3712fb7c4cbdb3567b5faf99bb4b 2024-12-05T11:16:31,280 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e0eb768fa03d45709fbf6036bd379169 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/B/e0eb768fa03d45709fbf6036bd379169 2024-12-05T11:16:31,284 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/29bdde8c169c4cfe806b67866ece94de to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/29bdde8c169c4cfe806b67866ece94de 2024-12-05T11:16:31,286 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/2d682d2cd3844555943f264b74e67f4f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/2d682d2cd3844555943f264b74e67f4f 2024-12-05T11:16:31,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9bddb9e85dbb47b3b7b73f4e72d73027 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/C/9bddb9e85dbb47b3b7b73f4e72d73027 2024-12-05T11:16:31,293 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/recovered.edits/456.seqid to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097/recovered.edits/456.seqid 2024-12-05T11:16:31,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/e19ca153de371b50b73536c04e3fa097 2024-12-05T11:16:31,294 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:16:31,297 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,300 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:16:31,303 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:16:31,304 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,305 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:16:31,305 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733397391305"}]},"ts":"9223372036854775807"} 2024-12-05T11:16:31,308 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:16:31,308 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e19ca153de371b50b73536c04e3fa097, NAME => 'TestAcidGuarantees,,1733397363977.e19ca153de371b50b73536c04e3fa097.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:16:31,308 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:16:31,308 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733397391308"}]},"ts":"9223372036854775807"} 2024-12-05T11:16:31,310 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:16:31,311 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-05T11:16:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T11:16:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:16:31,370 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:31,383 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 468), ProcessCount=11 (was 11), AvailableMemoryMB=5886 (was 6092) 2024-12-05T11:16:31,393 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=5885 2024-12-05T11:16:31,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:16:31,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:16:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:31,396 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T11:16:31,397 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:31,397 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-12-05T11:16:31,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T11:16:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-05T11:16:31,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742313_1489 (size=1061) 2024-12-05T11:16:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-05T11:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-05T11:16:31,855 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5 2024-12-05T11:16:31,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742314_1490 (size=53) 2024-12-05T11:16:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-05T11:16:32,093 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:16:32,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:16:32,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ed350aebe3c2ac2bb91697ce75c09c2f, disabling compactions & flushes 2024-12-05T11:16:32,261 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. after waiting 0 ms 2024-12-05T11:16:32,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,261 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed350aebe3c2ac2bb91697ce75c09c2f: Waiting for close lock at 1733397392261Disabling compacts and flushes for region at 1733397392261Disabling writes for close at 1733397392261Writing region close event to WAL at 1733397392261Closed at 1733397392261 2024-12-05T11:16:32,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T11:16:32,264 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733397392263"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733397392263"}]},"ts":"1733397392263"} 2024-12-05T11:16:32,266 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T11:16:32,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T11:16:32,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397392268"}]},"ts":"1733397392268"} 2024-12-05T11:16:32,271 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T11:16:32,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, ASSIGN}] 2024-12-05T11:16:32,273 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, ASSIGN 2024-12-05T11:16:32,274 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, ASSIGN; state=OFFLINE, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=false 2024-12-05T11:16:32,425 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:32,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, ASSIGN because future has completed 2024-12-05T11:16:32,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:16:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-05T11:16:32,592 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,592 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:16:32,593 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,593 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:16:32,593 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,593 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,594 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,596 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:32,596 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed350aebe3c2ac2bb91697ce75c09c2f columnFamilyName A 2024-12-05T11:16:32,596 DEBUG [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:32,596 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(327): Store=ed350aebe3c2ac2bb91697ce75c09c2f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:32,597 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,598 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:32,598 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed350aebe3c2ac2bb91697ce75c09c2f columnFamilyName B 2024-12-05T11:16:32,598 DEBUG [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:32,599 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(327): Store=ed350aebe3c2ac2bb91697ce75c09c2f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:32,599 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,600 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:32,600 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed350aebe3c2ac2bb91697ce75c09c2f columnFamilyName C 2024-12-05T11:16:32,600 DEBUG [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:32,601 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(327): Store=ed350aebe3c2ac2bb91697ce75c09c2f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:32,601 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,601 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,602 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,602 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,603 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,603 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,604 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:16:32,605 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,606 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T11:16:32,606 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened ed350aebe3c2ac2bb91697ce75c09c2f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60969102, jitterRate=-0.09148958325386047}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:16:32,606 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:32,607 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for ed350aebe3c2ac2bb91697ce75c09c2f: Running coprocessor pre-open hook at 1733397392593Writing region info on filesystem at 1733397392593Initializing all the Stores at 1733397392594 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397392594Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397392594Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397392594Cleaning up temporary data from old regions at 1733397392603 (+9 ms)Running coprocessor post-open hooks at 1733397392606 (+3 ms)Region opened successfully at 1733397392607 (+1 ms) 2024-12-05T11:16:32,607 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., pid=132, masterSystemTime=1733397392584 2024-12-05T11:16:32,609 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,609 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:32,610 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=OPEN, openSeqNum=2, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:32,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:16:32,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-05T11:16:32,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 in 182 msec 2024-12-05T11:16:32,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-05T11:16:32,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, ASSIGN in 342 msec 2024-12-05T11:16:32,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T11:16:32,615 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397392615"}]},"ts":"1733397392615"} 2024-12-05T11:16:32,616 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T11:16:32,616 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T11:16:32,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2210 sec 2024-12-05T11:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-05T11:16:33,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:33,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c89d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:33,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:33,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:33,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:33,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3117b36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:33,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388dd511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:33,569 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:33,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:33,572 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T11:16:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 6706a97ae65f,38049,1733397242931 2024-12-05T11:16:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e4cd910 2024-12-05T11:16:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T11:16:33,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T11:16:33,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T11:16:33,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T11:16:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T11:16:33,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742315_1491 (size=1097) 2024-12-05T11:16:33,992 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T11:16:33,992 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T11:16:33,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:16:34,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, REOPEN/MOVE}] 2024-12-05T11:16:34,003 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, REOPEN/MOVE 2024-12-05T11:16:34,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, REOPEN/MOVE because future has completed 2024-12-05T11:16:34,005 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:16:34,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:16:34,157 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,158 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:16:34,158 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing ed350aebe3c2ac2bb91697ce75c09c2f, disabling compactions & flushes 2024-12-05T11:16:34,158 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,158 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,158 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. after waiting 0 ms 2024-12-05T11:16:34,158 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,164 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T11:16:34,165 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,165 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for ed350aebe3c2ac2bb91697ce75c09c2f: Waiting for close lock at 1733397394158Running coprocessor pre-close hooks at 1733397394158Disabling compacts and flushes for region at 1733397394158Disabling writes for close at 1733397394158Writing region close event to WAL at 1733397394158Running coprocessor post-close hooks at 1733397394165 (+7 ms)Closed at 1733397394165 2024-12-05T11:16:34,165 WARN [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: ed350aebe3c2ac2bb91697ce75c09c2f to self. 2024-12-05T11:16:34,168 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,169 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=CLOSED 2024-12-05T11:16:34,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:16:34,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T11:16:34,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 in 168 msec 2024-12-05T11:16:34,177 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, REOPEN/MOVE; state=CLOSED, location=6706a97ae65f,42931,1733397243639; forceNewPlan=false, retain=true 2024-12-05T11:16:34,328 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=OPENING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, REOPEN/MOVE because future has completed 2024-12-05T11:16:34,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:16:34,530 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,530 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} 2024-12-05T11:16:34,530 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,530 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T11:16:34,531 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,531 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,532 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,533 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:34,533 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed350aebe3c2ac2bb91697ce75c09c2f columnFamilyName A 2024-12-05T11:16:34,535 DEBUG [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:34,536 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(327): Store=ed350aebe3c2ac2bb91697ce75c09c2f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:34,536 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,537 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:34,537 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed350aebe3c2ac2bb91697ce75c09c2f columnFamilyName B 2024-12-05T11:16:34,537 DEBUG [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:34,538 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(327): Store=ed350aebe3c2ac2bb91697ce75c09c2f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:34,538 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,539 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T11:16:34,539 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed350aebe3c2ac2bb91697ce75c09c2f columnFamilyName C 2024-12-05T11:16:34,539 DEBUG [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:34,540 INFO [StoreOpener-ed350aebe3c2ac2bb91697ce75c09c2f-1 {}] regionserver.HStore(327): Store=ed350aebe3c2ac2bb91697ce75c09c2f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T11:16:34,540 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,540 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,541 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,543 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,544 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,544 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,544 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T11:16:34,545 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,546 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened ed350aebe3c2ac2bb91697ce75c09c2f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60202300, jitterRate=-0.10291582345962524}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T11:16:34,546 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,546 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for ed350aebe3c2ac2bb91697ce75c09c2f: Running coprocessor pre-open hook at 1733397394531Writing region info on filesystem at 1733397394531Initializing all the Stores at 1733397394531Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397394532 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397394532Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733397394532Cleaning up temporary data from old regions at 1733397394544 (+12 ms)Running coprocessor post-open hooks at 1733397394546 (+2 ms)Region opened successfully at 1733397394546 2024-12-05T11:16:34,547 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., pid=137, masterSystemTime=1733397394527 2024-12-05T11:16:34,548 DEBUG [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,548 INFO [RS_OPEN_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,549 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=OPEN, openSeqNum=5, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:16:34,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-12-05T11:16:34,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 in 221 msec 2024-12-05T11:16:34,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-05T11:16:34,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, REOPEN/MOVE in 550 msec 2024-12-05T11:16:34,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-05T11:16:34,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 555 msec 2024-12-05T11:16:34,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 978 msec 2024-12-05T11:16:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T11:16:34,557 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:34,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfdfe02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676c4393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431c1422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fe6a55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ab35b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2113a512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784076c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,563 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5baab66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9aaf44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4612db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3537779d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56067e77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e024ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d3f17b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3e4b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf68053, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,571 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609105a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4b97d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a4260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5bb66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,576 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c48680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79dec48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,578 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62163bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5e986f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32717a8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,581 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1dd275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,582 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b2c542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9c29fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6706a97ae65f,38049,-1 for getting cluster id 2024-12-05T11:16:34,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T11:16:34,585 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b8d27a-9c74-4a66-9ada-fb2a65675e03' 2024-12-05T11:16:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T11:16:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b8d27a-9c74-4a66-9ada-fb2a65675e03" 2024-12-05T11:16:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575108aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6706a97ae65f,38049,-1] 2024-12-05T11:16:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T11:16:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:34,586 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T11:16:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc12f86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T11:16:34,587 DEBUG [Thread-2219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,587 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,587 DEBUG [Thread-2227 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,587 DEBUG [Thread-2208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,587 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,587 DEBUG [Thread-2221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,588 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,588 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,589 DEBUG [Thread-2225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,589 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-12-05T11:16:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T11:16:34,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,590 DEBUG [Thread-2223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T11:16:34,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,591 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:34,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6706a97ae65f,42931,1733397243639, seqNum=-1] 2024-12-05T11:16:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T11:16:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T11:16:34,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:34,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5] 2024-12-05T11:16:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:34,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:34,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:34,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120503884e0c1e3b4663be0637667c1a00b1_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397394598/Put/seqid=0 2024-12-05T11:16:34,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742316_1492 (size=12154) 2024-12-05T11:16:34,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397394604, max timestamp=1733397394635Num uniques 859; 2024-12-05T11:16:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397454634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397394604, max timestamp=1733397394635Num uniques 859; 2024-12-05T11:16:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733397394604, max timestamp=1733397394635Num uniques 859; 2024-12-05T11:16:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:34,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T11:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397454650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397454651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397454651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397454654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T11:16:34,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-05T11:16:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:34,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397454749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397454758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397454758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397454758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397454768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,836 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:16:34,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-05T11:16:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:34,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T11:16:34,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397454958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397454968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397454968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397454968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:34,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397454978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,019 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:35,022 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120503884e0c1e3b4663be0637667c1a00b1_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120503884e0c1e3b4663be0637667c1a00b1_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:35,023 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/7f36fe4270cf412bbd1c140ce2553161, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:35,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/7f36fe4270cf412bbd1c140ce2553161 is 175, key is test_row_0/A:col10/1733397394598/Put/seqid=0 2024-12-05T11:16:35,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742317_1493 (size=30955) 2024-12-05T11:16:35,027 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/7f36fe4270cf412bbd1c140ce2553161 2024-12-05T11:16:35,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/7239fa4abfb2417c9cd637faea39f8c0 is 50, key is test_row_0/B:col10/1733397394598/Put/seqid=0 2024-12-05T11:16:35,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742318_1494 (size=12001) 2024-12-05T11:16:35,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/7239fa4abfb2417c9cd637faea39f8c0 2024-12-05T11:16:35,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-05T11:16:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/0d2fdc67cc194964acb05128f5154bcc is 50, key is test_row_0/C:col10/1733397394598/Put/seqid=0 2024-12-05T11:16:35,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742319_1495 (size=12001) 2024-12-05T11:16:35,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-05T11:16:35,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:35,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T11:16:35,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397455269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397455278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397455278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397455278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397455288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-05T11:16:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:35,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/0d2fdc67cc194964acb05128f5154bcc 2024-12-05T11:16:35,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/7f36fe4270cf412bbd1c140ce2553161 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161 2024-12-05T11:16:35,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161, entries=150, sequenceid=15, filesize=30.2 K 2024-12-05T11:16:35,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/7239fa4abfb2417c9cd637faea39f8c0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/7239fa4abfb2417c9cd637faea39f8c0 2024-12-05T11:16:35,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/7239fa4abfb2417c9cd637faea39f8c0, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T11:16:35,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/0d2fdc67cc194964acb05128f5154bcc as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0d2fdc67cc194964acb05128f5154bcc 2024-12-05T11:16:35,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0d2fdc67cc194964acb05128f5154bcc, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T11:16:35,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ed350aebe3c2ac2bb91697ce75c09c2f in 881ms, sequenceid=15, compaction requested=false 2024-12-05T11:16:35,480 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T11:16:35,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:35,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-05T11:16:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:35,511 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-05T11:16:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:35,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058277e722529d49feb8ddaa9fa55f4601_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397394649/Put/seqid=0 2024-12-05T11:16:35,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742320_1496 (size=12154) 2024-12-05T11:16:35,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:35,528 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412058277e722529d49feb8ddaa9fa55f4601_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058277e722529d49feb8ddaa9fa55f4601_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6596a2435aca4798b53b400599ab2a32, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6596a2435aca4798b53b400599ab2a32 is 175, key is test_row_0/A:col10/1733397394649/Put/seqid=0 2024-12-05T11:16:35,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742321_1497 (size=30955) 2024-12-05T11:16:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T11:16:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:35,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:35,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397455791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397455792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397455793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397455794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397455798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397455898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397455898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397455908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397455909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:35,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6596a2435aca4798b53b400599ab2a32 2024-12-05T11:16:35,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/22b830811176442d86efef727cb74716 is 50, key is test_row_0/B:col10/1733397394649/Put/seqid=0 2024-12-05T11:16:35,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742322_1498 (size=12001) 2024-12-05T11:16:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397456108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397456108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397456118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397456118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,344 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/22b830811176442d86efef727cb74716 2024-12-05T11:16:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/111cd5d57c6b4d87bfe9ecc88cba4531 is 50, key is test_row_0/C:col10/1733397394649/Put/seqid=0 2024-12-05T11:16:36,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742323_1499 (size=12001) 2024-12-05T11:16:36,356 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/111cd5d57c6b4d87bfe9ecc88cba4531 2024-12-05T11:16:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6596a2435aca4798b53b400599ab2a32 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32 2024-12-05T11:16:36,362 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32, entries=150, sequenceid=50, filesize=30.2 K 2024-12-05T11:16:36,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/22b830811176442d86efef727cb74716 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/22b830811176442d86efef727cb74716 2024-12-05T11:16:36,366 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/22b830811176442d86efef727cb74716, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T11:16:36,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/111cd5d57c6b4d87bfe9ecc88cba4531 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/111cd5d57c6b4d87bfe9ecc88cba4531 2024-12-05T11:16:36,368 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/111cd5d57c6b4d87bfe9ecc88cba4531, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T11:16:36,369 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ed350aebe3c2ac2bb91697ce75c09c2f in 858ms, sequenceid=50, compaction requested=false 2024-12-05T11:16:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-12-05T11:16:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-12-05T11:16:36,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-05T11:16:36,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-12-05T11:16:36,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.7840 sec 2024-12-05T11:16:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:36,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T11:16:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:36,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050048fc00c9ea42fbacd18861aa88747a_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:36,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742324_1500 (size=14594) 2024-12-05T11:16:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397456440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397456442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397456443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397456444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397456548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397456558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397456558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397456558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T11:16:36,739 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:36,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-12-05T11:16:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T11:16:36,741 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:36,742 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:36,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397456758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397456768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397456768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397456768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397456818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:36,820 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:36,829 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:36,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050048fc00c9ea42fbacd18861aa88747a_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050048fc00c9ea42fbacd18861aa88747a_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:36,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d75927a4c47f421db0b18d746505c2b3, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:36,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d75927a4c47f421db0b18d746505c2b3 is 175, key is test_row_0/A:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:36,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742325_1501 (size=39549) 2024-12-05T11:16:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T11:16:36,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:36,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:36,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:36,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:36,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:36,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:37,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:37,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T11:16:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397457068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397457078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397457078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397457078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,236 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d75927a4c47f421db0b18d746505c2b3 2024-12-05T11:16:37,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8700a85e864b4571ab8d2001abf3ac5d is 50, key is test_row_0/B:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:37,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742326_1502 (size=12001) 2024-12-05T11:16:37,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8700a85e864b4571ab8d2001abf3ac5d 2024-12-05T11:16:37,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/3dece7e699c6426296242f18489e6d26 is 50, key is test_row_0/C:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:37,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742327_1503 (size=12001) 2024-12-05T11:16:37,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T11:16:37,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:37,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:37,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397457579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397457588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397457588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397457588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:37,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:37,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:37,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:37,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/3dece7e699c6426296242f18489e6d26 2024-12-05T11:16:37,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d75927a4c47f421db0b18d746505c2b3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3 2024-12-05T11:16:37,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3, entries=200, sequenceid=62, filesize=38.6 K 2024-12-05T11:16:37,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8700a85e864b4571ab8d2001abf3ac5d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8700a85e864b4571ab8d2001abf3ac5d 2024-12-05T11:16:37,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8700a85e864b4571ab8d2001abf3ac5d, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T11:16:37,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/3dece7e699c6426296242f18489e6d26 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/3dece7e699c6426296242f18489e6d26 2024-12-05T11:16:37,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/3dece7e699c6426296242f18489e6d26, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T11:16:37,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ed350aebe3c2ac2bb91697ce75c09c2f in 1255ms, sequenceid=62, compaction requested=true 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:37,675 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:37,675 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:37,676 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,676 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,676 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=99.1 K 2024-12-05T11:16:37,676 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/7239fa4abfb2417c9cd637faea39f8c0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/22b830811176442d86efef727cb74716, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8700a85e864b4571ab8d2001abf3ac5d] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.2 K 2024-12-05T11:16:37,676 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3] 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 7239fa4abfb2417c9cd637faea39f8c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733397394594 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f36fe4270cf412bbd1c140ce2553161, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733397394594 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b830811176442d86efef727cb74716, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733397394645 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6596a2435aca4798b53b400599ab2a32, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733397394645 2024-12-05T11:16:37,676 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 8700a85e864b4571ab8d2001abf3ac5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733397395792 2024-12-05T11:16:37,677 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d75927a4c47f421db0b18d746505c2b3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733397395791 2024-12-05T11:16:37,682 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:37,682 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:37,683 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/c5b77ab17cc64390b2da64d165549bf1 is 50, key is test_row_0/B:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:37,684 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205bff954e2c1fe426eb1091c3ed0271b9f_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:37,685 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205bff954e2c1fe426eb1091c3ed0271b9f_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:37,685 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bff954e2c1fe426eb1091c3ed0271b9f_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742328_1504 (size=12104) 2024-12-05T11:16:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742329_1505 (size=4469) 2024-12-05T11:16:37,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-05T11:16:37,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:37,811 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T11:16:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:37,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:37,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053c4a2ade273949d6920ac941e36f2400_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397396436/Put/seqid=0 2024-12-05T11:16:37,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742330_1506 (size=12154) 2024-12-05T11:16:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T11:16:38,093 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#427 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:38,093 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/f090d99765744dadb98d42595abe0b53 is 175, key is test_row_0/A:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:38,097 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/c5b77ab17cc64390b2da64d165549bf1 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c5b77ab17cc64390b2da64d165549bf1 2024-12-05T11:16:38,101 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into c5b77ab17cc64390b2da64d165549bf1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:38,101 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:38,101 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397397675; duration=0sec 2024-12-05T11:16:38,101 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:38,101 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:38,102 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:38,102 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:38,102 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:38,102 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:38,103 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0d2fdc67cc194964acb05128f5154bcc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/111cd5d57c6b4d87bfe9ecc88cba4531, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/3dece7e699c6426296242f18489e6d26] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.2 K 2024-12-05T11:16:38,103 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2fdc67cc194964acb05128f5154bcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733397394594 2024-12-05T11:16:38,103 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 111cd5d57c6b4d87bfe9ecc88cba4531, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733397394645 2024-12-05T11:16:38,104 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dece7e699c6426296242f18489e6d26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733397395792 2024-12-05T11:16:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742331_1507 (size=31058) 2024-12-05T11:16:38,110 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:38,111 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/0dcad02b0db74e6e9c97f30c8d033766 is 50, key is test_row_0/C:col10/1733397396419/Put/seqid=0 2024-12-05T11:16:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742332_1508 (size=12104) 2024-12-05T11:16:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:38,222 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053c4a2ade273949d6920ac941e36f2400_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c4a2ade273949d6920ac941e36f2400_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/eaa1119c390d4454bdba976f1d141aa7, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/eaa1119c390d4454bdba976f1d141aa7 is 175, key is test_row_0/A:col10/1733397396436/Put/seqid=0 2024-12-05T11:16:38,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742333_1509 (size=30955) 2024-12-05T11:16:38,226 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/eaa1119c390d4454bdba976f1d141aa7 2024-12-05T11:16:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/ded685274d3d4e4997f1aeceafba5064 is 50, key is test_row_0/B:col10/1733397396436/Put/seqid=0 2024-12-05T11:16:38,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742334_1510 (size=12001) 2024-12-05T11:16:38,240 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/ded685274d3d4e4997f1aeceafba5064 2024-12-05T11:16:38,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/72b93a58067e42be9e700b447cb75ff2 is 50, key is test_row_0/C:col10/1733397396436/Put/seqid=0 2024-12-05T11:16:38,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742335_1511 (size=12001) 2024-12-05T11:16:38,509 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/f090d99765744dadb98d42595abe0b53 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f090d99765744dadb98d42595abe0b53 2024-12-05T11:16:38,512 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into f090d99765744dadb98d42595abe0b53(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:38,513 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:38,513 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397397675; duration=0sec 2024-12-05T11:16:38,513 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:38,513 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:38,521 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/0dcad02b0db74e6e9c97f30c8d033766 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0dcad02b0db74e6e9c97f30c8d033766 2024-12-05T11:16:38,528 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into 0dcad02b0db74e6e9c97f30c8d033766(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:38,528 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:38,528 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397397675; duration=0sec 2024-12-05T11:16:38,528 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:38,529 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397458606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397458608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397458609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397458609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,610 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,610 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,650 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/72b93a58067e42be9e700b447cb75ff2 2024-12-05T11:16:38,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/eaa1119c390d4454bdba976f1d141aa7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7 2024-12-05T11:16:38,657 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7, entries=150, sequenceid=82, filesize=30.2 K 2024-12-05T11:16:38,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/ded685274d3d4e4997f1aeceafba5064 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/ded685274d3d4e4997f1aeceafba5064 2024-12-05T11:16:38,660 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/ded685274d3d4e4997f1aeceafba5064, entries=150, sequenceid=82, filesize=11.7 K 2024-12-05T11:16:38,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/72b93a58067e42be9e700b447cb75ff2 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/72b93a58067e42be9e700b447cb75ff2 2024-12-05T11:16:38,664 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/72b93a58067e42be9e700b447cb75ff2, entries=150, sequenceid=82, filesize=11.7 K 2024-12-05T11:16:38,664 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ed350aebe3c2ac2bb91697ce75c09c2f in 854ms, sequenceid=82, compaction requested=false 2024-12-05T11:16:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-12-05T11:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-12-05T11:16:38,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-05T11:16:38,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-12-05T11:16:38,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.9290 sec 2024-12-05T11:16:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:38,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T11:16:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:38,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053e97489dbff84555aad8555949d62865_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:38,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742336_1512 (size=14594) 2024-12-05T11:16:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397458744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397458747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397458838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,840 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397458858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397458858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-05T11:16:38,889 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:38,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-05T11:16:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T11:16:38,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:38,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:38,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T11:16:39,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T11:16:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397459068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:39,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397459068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:39,141 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:39,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053e97489dbff84555aad8555949d62865_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053e97489dbff84555aad8555949d62865_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:39,144 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/c8675dc89737482182832daab27b7b94, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:39,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/c8675dc89737482182832daab27b7b94 is 175, key is test_row_0/A:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:39,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742337_1513 (size=39549) 2024-12-05T11:16:39,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T11:16:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T11:16:39,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T11:16:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397459379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397459379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:39,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T11:16:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T11:16:39,549 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/c8675dc89737482182832daab27b7b94 2024-12-05T11:16:39,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/dacf38ecc4fd46c69a96362712455e94 is 50, key is test_row_0/B:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:39,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742338_1514 (size=12001) 2024-12-05T11:16:39,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/dacf38ecc4fd46c69a96362712455e94 2024-12-05T11:16:39,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/25640ecd99914c38b46d1cd864d4b817 is 50, key is test_row_0/C:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:39,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742339_1515 (size=12001) 2024-12-05T11:16:39,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/25640ecd99914c38b46d1cd864d4b817 2024-12-05T11:16:39,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/c8675dc89737482182832daab27b7b94 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94 2024-12-05T11:16:39,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94, entries=200, sequenceid=98, filesize=38.6 K 2024-12-05T11:16:39,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/dacf38ecc4fd46c69a96362712455e94 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/dacf38ecc4fd46c69a96362712455e94 2024-12-05T11:16:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/dacf38ecc4fd46c69a96362712455e94, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T11:16:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/25640ecd99914c38b46d1cd864d4b817 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/25640ecd99914c38b46d1cd864d4b817 2024-12-05T11:16:39,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/25640ecd99914c38b46d1cd864d4b817, entries=150, sequenceid=98, filesize=11.7 K 2024-12-05T11:16:39,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ed350aebe3c2ac2bb91697ce75c09c2f in 866ms, sequenceid=98, compaction requested=true 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:39,586 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:39,586 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:39,587 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,587 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,587 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f090d99765744dadb98d42595abe0b53, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=99.2 K 2024-12-05T11:16:39,587 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c5b77ab17cc64390b2da64d165549bf1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/ded685274d3d4e4997f1aeceafba5064, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/dacf38ecc4fd46c69a96362712455e94] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.3 K 2024-12-05T11:16:39,587 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f090d99765744dadb98d42595abe0b53, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94] 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b77ab17cc64390b2da64d165549bf1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733397395792 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f090d99765744dadb98d42595abe0b53, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733397395792 2024-12-05T11:16:39,587 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ded685274d3d4e4997f1aeceafba5064, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733397396436 2024-12-05T11:16:39,588 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa1119c390d4454bdba976f1d141aa7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733397396436 2024-12-05T11:16:39,588 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting dacf38ecc4fd46c69a96362712455e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733397398604 2024-12-05T11:16:39,588 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8675dc89737482182832daab27b7b94, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733397398604 2024-12-05T11:16:39,592 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:39,593 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:39,593 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/e6c291fbd12e421c8aa1e179e5b948a0 is 50, key is test_row_0/B:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:39,594 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205e7a0c854f5c94fadae6bd6f77a933f01_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:39,595 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205e7a0c854f5c94fadae6bd6f77a933f01_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:39,595 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e7a0c854f5c94fadae6bd6f77a933f01_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:39,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742340_1516 (size=12207) 2024-12-05T11:16:39,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742341_1517 (size=4469) 2024-12-05T11:16:39,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:39,654 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:39,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059a87b9e5b0d94a76abf7d73dcb07d75f_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397398745/Put/seqid=0 2024-12-05T11:16:39,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742342_1518 (size=12154) 2024-12-05T11:16:39,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:39,668 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059a87b9e5b0d94a76abf7d73dcb07d75f_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059a87b9e5b0d94a76abf7d73dcb07d75f_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/90294bf747d6443e858eae4a916c1b30, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/90294bf747d6443e858eae4a916c1b30 is 175, key is test_row_0/A:col10/1733397398745/Put/seqid=0 2024-12-05T11:16:39,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742343_1519 (size=30955) 2024-12-05T11:16:39,672 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/90294bf747d6443e858eae4a916c1b30 2024-12-05T11:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/516a5c6e6ad64061aa1b0099bf181bd6 is 50, key is test_row_0/B:col10/1733397398745/Put/seqid=0 2024-12-05T11:16:39,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742344_1520 (size=12001) 2024-12-05T11:16:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:39,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397459906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397459907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,013 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/e6c291fbd12e421c8aa1e179e5b948a0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/e6c291fbd12e421c8aa1e179e5b948a0 2024-12-05T11:16:40,013 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:40,014 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/90a52057dc084989b6be8d0110f02e09 is 175, key is test_row_0/A:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742345_1521 (size=31161) 2024-12-05T11:16:40,019 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into e6c291fbd12e421c8aa1e179e5b948a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:40,019 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:40,019 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397399586; duration=0sec 2024-12-05T11:16:40,020 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:40,020 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:40,020 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397460019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397460019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,021 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:40,022 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:40,022 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:40,022 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0dcad02b0db74e6e9c97f30c8d033766, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/72b93a58067e42be9e700b447cb75ff2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/25640ecd99914c38b46d1cd864d4b817] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.3 K 2024-12-05T11:16:40,022 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dcad02b0db74e6e9c97f30c8d033766, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733397395792 2024-12-05T11:16:40,022 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b93a58067e42be9e700b447cb75ff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733397396436 2024-12-05T11:16:40,023 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 25640ecd99914c38b46d1cd864d4b817, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733397398604 2024-12-05T11:16:40,024 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/90a52057dc084989b6be8d0110f02e09 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90a52057dc084989b6be8d0110f02e09 2024-12-05T11:16:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T11:16:40,029 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into 90a52057dc084989b6be8d0110f02e09(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:40,029 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:40,029 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397399586; duration=0sec 2024-12-05T11:16:40,029 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:40,029 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:40,032 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T11:16:40,034 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:40,035 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/c7f25f3c77c34c9682b18c1d61d8e0a0 is 50, key is test_row_0/C:col10/1733397398604/Put/seqid=0 2024-12-05T11:16:40,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742346_1522 (size=12207) 2024-12-05T11:16:40,062 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/c7f25f3c77c34c9682b18c1d61d8e0a0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c7f25f3c77c34c9682b18c1d61d8e0a0 2024-12-05T11:16:40,067 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into c7f25f3c77c34c9682b18c1d61d8e0a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:40,067 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:40,067 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397399586; duration=0sec 2024-12-05T11:16:40,067 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:40,067 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:40,080 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/516a5c6e6ad64061aa1b0099bf181bd6 2024-12-05T11:16:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/361e4ed4a7fc4da5a7411a30cf947528 is 50, key is test_row_0/C:col10/1733397398745/Put/seqid=0 2024-12-05T11:16:40,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742347_1523 (size=12001) 2024-12-05T11:16:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397460228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397460228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,498 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/361e4ed4a7fc4da5a7411a30cf947528 2024-12-05T11:16:40,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/90294bf747d6443e858eae4a916c1b30 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30 2024-12-05T11:16:40,504 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30, entries=150, sequenceid=117, filesize=30.2 K 2024-12-05T11:16:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/516a5c6e6ad64061aa1b0099bf181bd6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/516a5c6e6ad64061aa1b0099bf181bd6 2024-12-05T11:16:40,507 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/516a5c6e6ad64061aa1b0099bf181bd6, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T11:16:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/361e4ed4a7fc4da5a7411a30cf947528 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/361e4ed4a7fc4da5a7411a30cf947528 2024-12-05T11:16:40,511 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/361e4ed4a7fc4da5a7411a30cf947528, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T11:16:40,512 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ed350aebe3c2ac2bb91697ce75c09c2f in 858ms, sequenceid=117, compaction requested=false 2024-12-05T11:16:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-05T11:16:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-05T11:16:40,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-05T11:16:40,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-12-05T11:16:40,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.6260 sec 2024-12-05T11:16:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:40,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:16:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:40,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:40,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f5b44fd12fc244ef8cfc7fc24d04aa5c_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:40,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742348_1524 (size=17184) 2024-12-05T11:16:40,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397460567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397460567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397460618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,619 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397460628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,631 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397460678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397460679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397460888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397460888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:40,958 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:40,961 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f5b44fd12fc244ef8cfc7fc24d04aa5c_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f5b44fd12fc244ef8cfc7fc24d04aa5c_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:40,962 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/55a48973c5c241b38d68c62b4fb5081c, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:40,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/55a48973c5c241b38d68c62b4fb5081c is 175, key is test_row_0/A:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:40,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742349_1525 (size=48289) 2024-12-05T11:16:40,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/55a48973c5c241b38d68c62b4fb5081c 2024-12-05T11:16:40,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/1d37617b83544961ba810d0aeece1873 is 50, key is test_row_0/B:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:40,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742350_1526 (size=12101) 2024-12-05T11:16:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T11:16:41,039 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:41,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-05T11:16:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:16:41,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:41,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:41,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:16:41,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-05T11:16:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:41,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397461198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397461198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:41,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-05T11:16:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:16:41,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/1d37617b83544961ba810d0aeece1873 2024-12-05T11:16:41,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/ee26211990a84cebbb2cd21dae47fbce is 50, key is test_row_0/C:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:41,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742351_1527 (size=12101) 2024-12-05T11:16:41,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/ee26211990a84cebbb2cd21dae47fbce 2024-12-05T11:16:41,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/55a48973c5c241b38d68c62b4fb5081c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c 2024-12-05T11:16:41,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c, entries=250, sequenceid=135, filesize=47.2 K 2024-12-05T11:16:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/1d37617b83544961ba810d0aeece1873 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/1d37617b83544961ba810d0aeece1873 2024-12-05T11:16:41,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/1d37617b83544961ba810d0aeece1873, entries=150, sequenceid=135, filesize=11.8 K 2024-12-05T11:16:41,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/ee26211990a84cebbb2cd21dae47fbce as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ee26211990a84cebbb2cd21dae47fbce 2024-12-05T11:16:41,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ee26211990a84cebbb2cd21dae47fbce, entries=150, sequenceid=135, filesize=11.8 K 2024-12-05T11:16:41,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ed350aebe3c2ac2bb91697ce75c09c2f in 862ms, sequenceid=135, compaction requested=true 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:41,403 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:41,403 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:41,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:41,404 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:41,404 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:41,404 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,404 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90a52057dc084989b6be8d0110f02e09, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=107.8 K 2024-12-05T11:16:41,404 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,405 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90a52057dc084989b6be8d0110f02e09, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c] 2024-12-05T11:16:41,405 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:41,405 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:41,405 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,405 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/e6c291fbd12e421c8aa1e179e5b948a0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/516a5c6e6ad64061aa1b0099bf181bd6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/1d37617b83544961ba810d0aeece1873] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.5 K 2024-12-05T11:16:41,405 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90a52057dc084989b6be8d0110f02e09, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733397398604 2024-12-05T11:16:41,405 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c291fbd12e421c8aa1e179e5b948a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733397398604 2024-12-05T11:16:41,406 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90294bf747d6443e858eae4a916c1b30, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733397398723 2024-12-05T11:16:41,406 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 516a5c6e6ad64061aa1b0099bf181bd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733397398723 2024-12-05T11:16:41,407 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55a48973c5c241b38d68c62b4fb5081c, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733397399900 2024-12-05T11:16:41,407 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d37617b83544961ba810d0aeece1873, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733397399900 2024-12-05T11:16:41,413 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:41,416 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:41,416 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/6b11aa1d2084493389d128a70b21a9df is 50, key is test_row_0/B:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:41,417 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205172e1c2f798c4bc0a13692f8fd892d96_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:41,419 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205172e1c2f798c4bc0a13692f8fd892d96_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:41,419 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205172e1c2f798c4bc0a13692f8fd892d96_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:41,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742352_1528 (size=12409) 2024-12-05T11:16:41,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742353_1529 (size=4469) 2024-12-05T11:16:41,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,499 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057e2fd6011c294c8188743a4b2d981bd5_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397400561/Put/seqid=0 2024-12-05T11:16:41,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742354_1530 (size=12304) 2024-12-05T11:16:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:41,517 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057e2fd6011c294c8188743a4b2d981bd5_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057e2fd6011c294c8188743a4b2d981bd5_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/b933af4fbdf8485b9b148a5d4ed02bad, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/b933af4fbdf8485b9b148a5d4ed02bad is 175, key is test_row_0/A:col10/1733397400561/Put/seqid=0 2024-12-05T11:16:41,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742355_1531 (size=31105) 2024-12-05T11:16:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:16:41,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:41,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397461726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:41,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397461727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:41,829 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/6b11aa1d2084493389d128a70b21a9df as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6b11aa1d2084493389d128a70b21a9df 2024-12-05T11:16:41,830 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#444 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:41,831 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/15244aa119044a16a14bcd9bb1eed8aa is 175, key is test_row_0/A:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:41,833 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into 6b11aa1d2084493389d128a70b21a9df(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:41,833 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:41,833 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397401403; duration=0sec 2024-12-05T11:16:41,833 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:41,833 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:41,833 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742356_1532 (size=31363) 2024-12-05T11:16:41,834 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:41,834 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:41,834 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,834 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c7f25f3c77c34c9682b18c1d61d8e0a0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/361e4ed4a7fc4da5a7411a30cf947528, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ee26211990a84cebbb2cd21dae47fbce] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.5 K 2024-12-05T11:16:41,834 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f25f3c77c34c9682b18c1d61d8e0a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733397398604 2024-12-05T11:16:41,835 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 361e4ed4a7fc4da5a7411a30cf947528, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733397398723 2024-12-05T11:16:41,835 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ee26211990a84cebbb2cd21dae47fbce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733397399900 2024-12-05T11:16:41,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397461838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:41,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397461838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:41,841 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:41,841 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/f8c6d7956f604263bd61ec06d951bdf0 is 50, key is test_row_0/C:col10/1733397400539/Put/seqid=0 2024-12-05T11:16:41,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742357_1533 (size=12409) 2024-12-05T11:16:41,923 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/b933af4fbdf8485b9b148a5d4ed02bad 2024-12-05T11:16:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/40204c71536a48a8bebdf748600c88ee is 50, key is test_row_0/B:col10/1733397400561/Put/seqid=0 2024-12-05T11:16:41,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742358_1534 (size=12151) 2024-12-05T11:16:41,931 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/40204c71536a48a8bebdf748600c88ee 2024-12-05T11:16:41,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1eccbfa91282496b98b707bbe8a0b882 is 50, key is test_row_0/C:col10/1733397400561/Put/seqid=0 2024-12-05T11:16:41,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742359_1535 (size=12151) 2024-12-05T11:16:41,939 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1eccbfa91282496b98b707bbe8a0b882 2024-12-05T11:16:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/b933af4fbdf8485b9b148a5d4ed02bad as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad 2024-12-05T11:16:41,946 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad, entries=150, sequenceid=152, filesize=30.4 K 2024-12-05T11:16:41,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/40204c71536a48a8bebdf748600c88ee as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/40204c71536a48a8bebdf748600c88ee 2024-12-05T11:16:41,950 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/40204c71536a48a8bebdf748600c88ee, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T11:16:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1eccbfa91282496b98b707bbe8a0b882 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1eccbfa91282496b98b707bbe8a0b882 2024-12-05T11:16:41,953 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1eccbfa91282496b98b707bbe8a0b882, entries=150, sequenceid=152, filesize=11.9 K 2024-12-05T11:16:41,953 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ed350aebe3c2ac2bb91697ce75c09c2f in 454ms, sequenceid=152, compaction requested=false 2024-12-05T11:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-05T11:16:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-05T11:16:41,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-05T11:16:41,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 913 msec 2024-12-05T11:16:41,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 918 msec 2024-12-05T11:16:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:42,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T11:16:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:42,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fbcdc1e3a34e4fd0904223eb7894d4ce_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:42,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742360_1536 (size=17284) 2024-12-05T11:16:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397462076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397462078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T11:16:42,178 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:42,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-05T11:16:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:42,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:42,181 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:42,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:42,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397462188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397462188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,238 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/15244aa119044a16a14bcd9bb1eed8aa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/15244aa119044a16a14bcd9bb1eed8aa 2024-12-05T11:16:42,241 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into 15244aa119044a16a14bcd9bb1eed8aa(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:42,241 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:42,241 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397401403; duration=0sec 2024-12-05T11:16:42,241 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:42,241 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:42,247 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/f8c6d7956f604263bd61ec06d951bdf0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/f8c6d7956f604263bd61ec06d951bdf0 2024-12-05T11:16:42,250 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into f8c6d7956f604263bd61ec06d951bdf0(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:42,250 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:42,250 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397401403; duration=0sec 2024-12-05T11:16:42,250 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:42,250 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:42,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397462398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397462398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:42,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fbcdc1e3a34e4fd0904223eb7894d4ce_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fbcdc1e3a34e4fd0904223eb7894d4ce_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:42,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/1763b10dd5874cd4bc3239b678901ec4, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:42,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/1763b10dd5874cd4bc3239b678901ec4 is 175, key is test_row_0/A:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742361_1537 (size=48389) 2024-12-05T11:16:42,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:42,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:42,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:42,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397462708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397462708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:42,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:42,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:42,869 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/1763b10dd5874cd4bc3239b678901ec4 2024-12-05T11:16:42,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/f957767d7f044eb6b4901e2dd460dbca is 50, key is test_row_0/B:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:42,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742362_1538 (size=12151) 2024-12-05T11:16:42,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397462888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:42,890 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:42,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:42,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:42,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:42,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:43,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:43,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397463218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:43,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397463218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:43,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:43,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:43,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/f957767d7f044eb6b4901e2dd460dbca 2024-12-05T11:16:43,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/d11178f0712b4c7e9f63a94cc0e9dde3 is 50, key is test_row_0/C:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742363_1539 (size=12151) 2024-12-05T11:16:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:43,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:43,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:43,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:43,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:43,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:43,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/d11178f0712b4c7e9f63a94cc0e9dde3 2024-12-05T11:16:43,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/1763b10dd5874cd4bc3239b678901ec4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4 2024-12-05T11:16:43,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4, entries=250, sequenceid=170, filesize=47.3 K 2024-12-05T11:16:43,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/f957767d7f044eb6b4901e2dd460dbca as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/f957767d7f044eb6b4901e2dd460dbca 2024-12-05T11:16:43,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/f957767d7f044eb6b4901e2dd460dbca, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T11:16:43,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/d11178f0712b4c7e9f63a94cc0e9dde3 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/d11178f0712b4c7e9f63a94cc0e9dde3 2024-12-05T11:16:43,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/d11178f0712b4c7e9f63a94cc0e9dde3, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T11:16:43,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ed350aebe3c2ac2bb91697ce75c09c2f in 1652ms, sequenceid=170, compaction requested=true 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:43,704 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:43,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:43,704 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:43,705 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,705 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,705 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/15244aa119044a16a14bcd9bb1eed8aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=108.3 K 2024-12-05T11:16:43,705 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6b11aa1d2084493389d128a70b21a9df, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/40204c71536a48a8bebdf748600c88ee, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/f957767d7f044eb6b4901e2dd460dbca] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.9 K 2024-12-05T11:16:43,705 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/15244aa119044a16a14bcd9bb1eed8aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4] 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b11aa1d2084493389d128a70b21a9df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733397399900 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15244aa119044a16a14bcd9bb1eed8aa, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733397399900 2024-12-05T11:16:43,705 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 40204c71536a48a8bebdf748600c88ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733397400561 2024-12-05T11:16:43,706 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting b933af4fbdf8485b9b148a5d4ed02bad, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733397400561 2024-12-05T11:16:43,706 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f957767d7f044eb6b4901e2dd460dbca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733397401723 2024-12-05T11:16:43,706 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1763b10dd5874cd4bc3239b678901ec4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733397401709 2024-12-05T11:16:43,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T11:16:43,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,708 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T11:16:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:43,711 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:43,712 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:43,712 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/c411bad288154dee819b32f47ba32089 is 50, key is test_row_0/B:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:43,717 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412059b3e355055ce4e4696eea00d490765f3_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:43,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c62d400b80374fa188a785cd84bcdc9f_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397402077/Put/seqid=0 2024-12-05T11:16:43,719 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412059b3e355055ce4e4696eea00d490765f3_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:43,719 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b3e355055ce4e4696eea00d490765f3_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742364_1540 (size=12561) 2024-12-05T11:16:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742365_1541 (size=12304) 2024-12-05T11:16:43,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742366_1542 (size=4469) 2024-12-05T11:16:43,724 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#454 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:43,724 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/bf510443f65441bba4f7e597384dd7aa is 175, key is test_row_0/A:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:43,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742367_1543 (size=31515) 2024-12-05T11:16:43,727 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/c411bad288154dee819b32f47ba32089 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c411bad288154dee819b32f47ba32089 2024-12-05T11:16:43,730 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into c411bad288154dee819b32f47ba32089(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:43,730 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:43,730 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397403704; duration=0sec 2024-12-05T11:16:43,730 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:43,730 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:43,731 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:43,731 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:43,731 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:43,731 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:43,731 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/f8c6d7956f604263bd61ec06d951bdf0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1eccbfa91282496b98b707bbe8a0b882, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/d11178f0712b4c7e9f63a94cc0e9dde3] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=35.9 K 2024-12-05T11:16:43,732 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c6d7956f604263bd61ec06d951bdf0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733397399900 2024-12-05T11:16:43,732 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eccbfa91282496b98b707bbe8a0b882, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733397400561 2024-12-05T11:16:43,733 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting d11178f0712b4c7e9f63a94cc0e9dde3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733397401723 2024-12-05T11:16:43,738 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:43,738 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/b1b35826144340c38169b20227c56dc7 is 50, key is test_row_0/C:col10/1733397402050/Put/seqid=0 2024-12-05T11:16:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742368_1544 (size=12561) 2024-12-05T11:16:43,745 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/b1b35826144340c38169b20227c56dc7 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/b1b35826144340c38169b20227c56dc7 2024-12-05T11:16:43,749 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into b1b35826144340c38169b20227c56dc7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:43,749 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:43,749 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397403704; duration=0sec 2024-12-05T11:16:43,749 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:43,749 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:44,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:44,125 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c62d400b80374fa188a785cd84bcdc9f_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c62d400b80374fa188a785cd84bcdc9f_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:44,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/88d014dc2a0c4ee8aede9b8c8a411548, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:44,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/88d014dc2a0c4ee8aede9b8c8a411548 is 175, key is test_row_0/A:col10/1733397402077/Put/seqid=0 2024-12-05T11:16:44,130 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/bf510443f65441bba4f7e597384dd7aa as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/bf510443f65441bba4f7e597384dd7aa 2024-12-05T11:16:44,134 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into bf510443f65441bba4f7e597384dd7aa(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:44,134 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:44,134 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397403704; duration=0sec 2024-12-05T11:16:44,134 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:44,134 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:44,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742369_1545 (size=31105) 2024-12-05T11:16:44,144 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/88d014dc2a0c4ee8aede9b8c8a411548 2024-12-05T11:16:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/af1c6cf6614a4f1fa2ad5764a9311558 is 50, key is test_row_0/B:col10/1733397402077/Put/seqid=0 2024-12-05T11:16:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742370_1546 (size=12151) 2024-12-05T11:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:44,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397464251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397464254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:44,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397464359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397464368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,553 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/af1c6cf6614a4f1fa2ad5764a9311558 2024-12-05T11:16:44,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/5898f0ebd94b484781e91d985c0fc1b4 is 50, key is test_row_0/C:col10/1733397402077/Put/seqid=0 2024-12-05T11:16:44,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742371_1547 (size=12151) 2024-12-05T11:16:44,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397464568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397464579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58206 deadline: 1733397464638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,640 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_0' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58254 deadline: 1733397464679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,680 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397464878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397464888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:44,966 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/5898f0ebd94b484781e91d985c0fc1b4 2024-12-05T11:16:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/88d014dc2a0c4ee8aede9b8c8a411548 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548 2024-12-05T11:16:44,977 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548, entries=150, sequenceid=187, filesize=30.4 K 2024-12-05T11:16:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/af1c6cf6614a4f1fa2ad5764a9311558 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/af1c6cf6614a4f1fa2ad5764a9311558 2024-12-05T11:16:44,985 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/af1c6cf6614a4f1fa2ad5764a9311558, entries=150, sequenceid=187, filesize=11.9 K 2024-12-05T11:16:44,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/5898f0ebd94b484781e91d985c0fc1b4 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5898f0ebd94b484781e91d985c0fc1b4 2024-12-05T11:16:44,989 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5898f0ebd94b484781e91d985c0fc1b4, entries=150, sequenceid=187, filesize=11.9 K 2024-12-05T11:16:44,989 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ed350aebe3c2ac2bb91697ce75c09c2f in 1282ms, sequenceid=187, compaction requested=false 2024-12-05T11:16:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-05T11:16:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-05T11:16:44,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-05T11:16:44,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8100 sec 2024-12-05T11:16:44,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.8150 sec 2024-12-05T11:16:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:45,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T11:16:45,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:45,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:45,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:45,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:45,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:45,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:45,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205edead28e4de14e498f61da0690faf59e_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:45,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742372_1548 (size=14794) 2024-12-05T11:16:45,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397465416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397465416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397465529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397465529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397465738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397465738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:45,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:45,803 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205edead28e4de14e498f61da0690faf59e_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205edead28e4de14e498f61da0690faf59e_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:45,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/2bd1e85993134d3a9a83da84a0446232, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:45,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/2bd1e85993134d3a9a83da84a0446232 is 175, key is test_row_0/A:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742373_1549 (size=39749) 2024-12-05T11:16:46,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397466048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:46,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:46,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397466048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:46,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:46,207 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/2bd1e85993134d3a9a83da84a0446232 2024-12-05T11:16:46,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/cec09162effd4f03b2417c751592a4df is 50, key is test_row_0/B:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:46,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742374_1550 (size=12151) 2024-12-05T11:16:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T11:16:46,348 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:46,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-05T11:16:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T11:16:46,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:46,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:46,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T11:16:46,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T11:16:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:46,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397466558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397466558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:46,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/cec09162effd4f03b2417c751592a4df 2024-12-05T11:16:46,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/6f685fdc55354581bedb1c856068dfeb is 50, key is test_row_0/C:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:46,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742375_1551 (size=12151) 2024-12-05T11:16:46,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/6f685fdc55354581bedb1c856068dfeb 2024-12-05T11:16:46,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/2bd1e85993134d3a9a83da84a0446232 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232 2024-12-05T11:16:46,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232, entries=200, sequenceid=208, filesize=38.8 K 2024-12-05T11:16:46,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/cec09162effd4f03b2417c751592a4df as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/cec09162effd4f03b2417c751592a4df 2024-12-05T11:16:46,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/cec09162effd4f03b2417c751592a4df, entries=150, sequenceid=208, filesize=11.9 K 2024-12-05T11:16:46,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/6f685fdc55354581bedb1c856068dfeb as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6f685fdc55354581bedb1c856068dfeb 2024-12-05T11:16:46,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6f685fdc55354581bedb1c856068dfeb, entries=150, sequenceid=208, filesize=11.9 K 2024-12-05T11:16:46,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ed350aebe3c2ac2bb91697ce75c09c2f in 1253ms, sequenceid=208, compaction requested=true 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:46,646 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:46,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:46,646 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:46,647 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:46,647 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:46,647 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:46,647 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:46,647 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,647 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,647 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c411bad288154dee819b32f47ba32089, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/af1c6cf6614a4f1fa2ad5764a9311558, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/cec09162effd4f03b2417c751592a4df] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.0 K 2024-12-05T11:16:46,647 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/bf510443f65441bba4f7e597384dd7aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=100.0 K 2024-12-05T11:16:46,647 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,647 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/bf510443f65441bba4f7e597384dd7aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232] 2024-12-05T11:16:46,647 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c411bad288154dee819b32f47ba32089, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733397401723 2024-12-05T11:16:46,648 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf510443f65441bba4f7e597384dd7aa, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733397401723 2024-12-05T11:16:46,648 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting af1c6cf6614a4f1fa2ad5764a9311558, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733397402056 2024-12-05T11:16:46,648 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d014dc2a0c4ee8aede9b8c8a411548, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733397402056 2024-12-05T11:16:46,648 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting cec09162effd4f03b2417c751592a4df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733397404249 2024-12-05T11:16:46,648 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd1e85993134d3a9a83da84a0446232, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733397404249 2024-12-05T11:16:46,654 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:46,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T11:16:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,655 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T11:16:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:46,656 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205db32d25c45fd4fab976fe421a1225544_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:46,656 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:46,657 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/b7d771db7ab441399c2597fd25dfaf70 is 50, key is test_row_0/B:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:46,658 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205db32d25c45fd4fab976fe421a1225544_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:46,658 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205db32d25c45fd4fab976fe421a1225544_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T11:16:46,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742377_1553 (size=4469) 2024-12-05T11:16:46,672 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#462 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:46,672 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/f90a5a746708428493a46953a1ec23f9 is 175, key is test_row_0/A:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742376_1552 (size=12663) 2024-12-05T11:16:46,679 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/b7d771db7ab441399c2597fd25dfaf70 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b7d771db7ab441399c2597fd25dfaf70 2024-12-05T11:16:46,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205249bf2a6364c4d80ba1f1f3aad30b1c6_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397405410/Put/seqid=0 2024-12-05T11:16:46,685 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into b7d771db7ab441399c2597fd25dfaf70(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:46,685 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:46,686 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397406646; duration=0sec 2024-12-05T11:16:46,686 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:46,686 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:46,686 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:46,687 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:46,687 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:46,687 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:46,687 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/b1b35826144340c38169b20227c56dc7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5898f0ebd94b484781e91d985c0fc1b4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6f685fdc55354581bedb1c856068dfeb] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.0 K 2024-12-05T11:16:46,687 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b35826144340c38169b20227c56dc7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733397401723 2024-12-05T11:16:46,688 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5898f0ebd94b484781e91d985c0fc1b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733397402056 2024-12-05T11:16:46,688 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f685fdc55354581bedb1c856068dfeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733397404249 2024-12-05T11:16:46,702 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:46,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742379_1555 (size=12304) 2024-12-05T11:16:46,702 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/ce8a9c9b55a64498b370f379e72e1595 is 50, key is test_row_0/C:col10/1733397404250/Put/seqid=0 2024-12-05T11:16:46,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742378_1554 (size=31617) 2024-12-05T11:16:46,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:46,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742380_1556 (size=12663) 2024-12-05T11:16:46,710 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205249bf2a6364c4d80ba1f1f3aad30b1c6_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205249bf2a6364c4d80ba1f1f3aad30b1c6_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d401cd9df10a464babc498383b436554, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d401cd9df10a464babc498383b436554 is 175, key is test_row_0/A:col10/1733397405410/Put/seqid=0 2024-12-05T11:16:46,712 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/f90a5a746708428493a46953a1ec23f9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f90a5a746708428493a46953a1ec23f9 2024-12-05T11:16:46,713 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/ce8a9c9b55a64498b370f379e72e1595 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ce8a9c9b55a64498b370f379e72e1595 2024-12-05T11:16:46,718 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into f90a5a746708428493a46953a1ec23f9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:46,718 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:46,718 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397406646; duration=0sec 2024-12-05T11:16:46,718 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:46,718 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:46,718 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into ce8a9c9b55a64498b370f379e72e1595(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:46,718 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:46,718 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397406646; duration=0sec 2024-12-05T11:16:46,718 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:46,718 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:46,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742381_1557 (size=31105) 2024-12-05T11:16:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T11:16:47,122 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d401cd9df10a464babc498383b436554 2024-12-05T11:16:47,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/6201db35fcfe4e9faf1a43989fed4c0d is 50, key is test_row_0/B:col10/1733397405410/Put/seqid=0 2024-12-05T11:16:47,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742382_1558 (size=12151) 2024-12-05T11:16:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T11:16:47,535 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/6201db35fcfe4e9faf1a43989fed4c0d 2024-12-05T11:16:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/c3eece5250e1458cae86443d607f8a66 is 50, key is test_row_0/C:col10/1733397405410/Put/seqid=0 2024-12-05T11:16:47,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742383_1559 (size=12151) 2024-12-05T11:16:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:47,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397467614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397467615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:47,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397467728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:47,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397467728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397467938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397467938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:47,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:47,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:47,945 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/c3eece5250e1458cae86443d607f8a66 2024-12-05T11:16:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d401cd9df10a464babc498383b436554 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554 2024-12-05T11:16:47,950 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554, entries=150, sequenceid=222, filesize=30.4 K 2024-12-05T11:16:47,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/6201db35fcfe4e9faf1a43989fed4c0d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6201db35fcfe4e9faf1a43989fed4c0d 2024-12-05T11:16:47,955 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6201db35fcfe4e9faf1a43989fed4c0d, entries=150, sequenceid=222, filesize=11.9 K 2024-12-05T11:16:47,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/c3eece5250e1458cae86443d607f8a66 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c3eece5250e1458cae86443d607f8a66 2024-12-05T11:16:47,959 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c3eece5250e1458cae86443d607f8a66, entries=150, sequenceid=222, filesize=11.9 K 2024-12-05T11:16:47,960 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ed350aebe3c2ac2bb91697ce75c09c2f in 1305ms, sequenceid=222, compaction requested=false 2024-12-05T11:16:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-05T11:16:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-05T11:16:47,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-05T11:16:47,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-12-05T11:16:47,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.6140 sec 2024-12-05T11:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:48,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:48,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d02ed50d3a1244aaa3310f2467f86f96_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:48,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742384_1560 (size=14794) 2024-12-05T11:16:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397468264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397468266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397468379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397468379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T11:16:48,499 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:48,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-05T11:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:48,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:48,502 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:48,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397468588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397468588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:48,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:48,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:48,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d02ed50d3a1244aaa3310f2467f86f96_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d02ed50d3a1244aaa3310f2467f86f96_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:48,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d4cbe082889d43fa86fd3c9b7de3fb05, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:48,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d4cbe082889d43fa86fd3c9b7de3fb05 is 175, key is test_row_0/A:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:48,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742385_1561 (size=39749) 2024-12-05T11:16:48,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:48,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:48,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:48,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:48,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:48,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397468898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397468898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:48,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:48,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,066 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d4cbe082889d43fa86fd3c9b7de3fb05 2024-12-05T11:16:49,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8af4344abb9142d1a7fdc027bc97b09c is 50, key is test_row_0/B:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:49,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742386_1562 (size=12151) 2024-12-05T11:16:49,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:49,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:49,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:49,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397469408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397469408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:49,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8af4344abb9142d1a7fdc027bc97b09c 2024-12-05T11:16:49,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/462c1adcba22400bb6e41eb5405b3425 is 50, key is test_row_0/C:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:49,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742387_1563 (size=12151) 2024-12-05T11:16:49,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:49,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:49,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:49,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:49,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/462c1adcba22400bb6e41eb5405b3425 2024-12-05T11:16:49,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/d4cbe082889d43fa86fd3c9b7de3fb05 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05 2024-12-05T11:16:49,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05, entries=200, sequenceid=245, filesize=38.8 K 2024-12-05T11:16:49,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8af4344abb9142d1a7fdc027bc97b09c as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8af4344abb9142d1a7fdc027bc97b09c 2024-12-05T11:16:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8af4344abb9142d1a7fdc027bc97b09c, entries=150, sequenceid=245, filesize=11.9 K 2024-12-05T11:16:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/462c1adcba22400bb6e41eb5405b3425 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/462c1adcba22400bb6e41eb5405b3425 2024-12-05T11:16:49,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/462c1adcba22400bb6e41eb5405b3425, entries=150, sequenceid=245, filesize=11.9 K 2024-12-05T11:16:49,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ed350aebe3c2ac2bb91697ce75c09c2f in 1646ms, sequenceid=245, compaction requested=true 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:49,897 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:49,897 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:49,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:49,897 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:49,897 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:49,897 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:49,897 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:49,897 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,897 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,898 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b7d771db7ab441399c2597fd25dfaf70, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6201db35fcfe4e9faf1a43989fed4c0d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8af4344abb9142d1a7fdc027bc97b09c] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.1 K 2024-12-05T11:16:49,898 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f90a5a746708428493a46953a1ec23f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=100.1 K 2024-12-05T11:16:49,898 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,898 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f90a5a746708428493a46953a1ec23f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05] 2024-12-05T11:16:49,898 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d771db7ab441399c2597fd25dfaf70, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733397404249 2024-12-05T11:16:49,898 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6201db35fcfe4e9faf1a43989fed4c0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397405410 2024-12-05T11:16:49,898 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting f90a5a746708428493a46953a1ec23f9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733397404249 2024-12-05T11:16:49,899 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d401cd9df10a464babc498383b436554, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397405410 2024-12-05T11:16:49,899 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 8af4344abb9142d1a7fdc027bc97b09c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397407600 2024-12-05T11:16:49,899 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4cbe082889d43fa86fd3c9b7de3fb05, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397407594 2024-12-05T11:16:49,903 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:49,904 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:49,904 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/c39e5daedfb44728b5feb7f3152fefbd is 50, key is test_row_0/B:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:49,905 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120555cf9421276e42e08637dd7e3de5be50_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:49,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742388_1564 (size=12765) 2024-12-05T11:16:49,907 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120555cf9421276e42e08637dd7e3de5be50_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:49,908 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120555cf9421276e42e08637dd7e3de5be50_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:49,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742389_1565 (size=4469) 2024-12-05T11:16:49,918 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/c39e5daedfb44728b5feb7f3152fefbd as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c39e5daedfb44728b5feb7f3152fefbd 2024-12-05T11:16:49,922 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into c39e5daedfb44728b5feb7f3152fefbd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:49,922 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:49,922 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397409897; duration=0sec 2024-12-05T11:16:49,922 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:49,922 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:49,922 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:49,922 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:49,923 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:49,923 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:49,923 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ce8a9c9b55a64498b370f379e72e1595, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c3eece5250e1458cae86443d607f8a66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/462c1adcba22400bb6e41eb5405b3425] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.1 K 2024-12-05T11:16:49,923 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting ce8a9c9b55a64498b370f379e72e1595, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733397404249 2024-12-05T11:16:49,923 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c3eece5250e1458cae86443d607f8a66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733397405410 2024-12-05T11:16:49,923 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 462c1adcba22400bb6e41eb5405b3425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397407600 2024-12-05T11:16:49,929 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#473 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:49,929 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/5299caecce9146beb96912dac74cef88 is 50, key is test_row_0/C:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:49,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742390_1566 (size=12765) 2024-12-05T11:16:50,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T11:16:50,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:50,028 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T11:16:50,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:50,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:50,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205938251cfc4354b5cb5d72af43fb1f4ef_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397408260/Put/seqid=0 2024-12-05T11:16:50,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742391_1567 (size=12304) 2024-12-05T11:16:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:50,040 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205938251cfc4354b5cb5d72af43fb1f4ef_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205938251cfc4354b5cb5d72af43fb1f4ef_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:50,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/077f1842fead4d37982a543734a80e08, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/077f1842fead4d37982a543734a80e08 is 175, key is test_row_0/A:col10/1733397408260/Put/seqid=0 2024-12-05T11:16:50,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742392_1568 (size=31105) 2024-12-05T11:16:50,319 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#472 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:50,319 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/eda73d122c1848debed0ea9c69137e16 is 175, key is test_row_0/A:col10/1733397407600/Put/seqid=0 2024-12-05T11:16:50,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742393_1569 (size=31719) 2024-12-05T11:16:50,335 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/5299caecce9146beb96912dac74cef88 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5299caecce9146beb96912dac74cef88 2024-12-05T11:16:50,338 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into 5299caecce9146beb96912dac74cef88(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:50,338 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:50,338 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397409897; duration=0sec 2024-12-05T11:16:50,338 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:50,338 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:50,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/077f1842fead4d37982a543734a80e08 2024-12-05T11:16:50,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/384932f273d849ec8a00bc31153640c6 is 50, key is test_row_0/B:col10/1733397408260/Put/seqid=0 2024-12-05T11:16:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742394_1570 (size=12151) 2024-12-05T11:16:50,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397470454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:50,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:50,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397470457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:50,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397470568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397470568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:50,726 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/eda73d122c1848debed0ea9c69137e16 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eda73d122c1848debed0ea9c69137e16 2024-12-05T11:16:50,729 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into eda73d122c1848debed0ea9c69137e16(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:50,729 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:50,729 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397409897; duration=0sec 2024-12-05T11:16:50,729 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:50,729 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397470779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397470779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:50,856 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/384932f273d849ec8a00bc31153640c6 2024-12-05T11:16:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1906e1bae13a4661b3600973b88bc066 is 50, key is test_row_0/C:col10/1733397408260/Put/seqid=0 2024-12-05T11:16:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742395_1571 (size=12151) 2024-12-05T11:16:51,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397471088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397471088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,271 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1906e1bae13a4661b3600973b88bc066 2024-12-05T11:16:51,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/077f1842fead4d37982a543734a80e08 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08 2024-12-05T11:16:51,277 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08, entries=150, sequenceid=258, filesize=30.4 K 2024-12-05T11:16:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/384932f273d849ec8a00bc31153640c6 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/384932f273d849ec8a00bc31153640c6 2024-12-05T11:16:51,284 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/384932f273d849ec8a00bc31153640c6, entries=150, sequenceid=258, filesize=11.9 K 2024-12-05T11:16:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1906e1bae13a4661b3600973b88bc066 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1906e1bae13a4661b3600973b88bc066 2024-12-05T11:16:51,288 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1906e1bae13a4661b3600973b88bc066, entries=150, sequenceid=258, filesize=11.9 K 2024-12-05T11:16:51,288 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ed350aebe3c2ac2bb91697ce75c09c2f in 1260ms, sequenceid=258, compaction requested=false 2024-12-05T11:16:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-05T11:16:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-05T11:16:51,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-05T11:16:51,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7880 sec 2024-12-05T11:16:51,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.7930 sec 2024-12-05T11:16:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:51,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T11:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:51,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T11:16:51,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054efeedbee3ca4964a46ee763d8d5ec90_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742396_1572 (size=14994) 2024-12-05T11:16:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397471613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397471616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397471718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397471729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397471928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397471938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:52,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:52,012 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054efeedbee3ca4964a46ee763d8d5ec90_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054efeedbee3ca4964a46ee763d8d5ec90_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:52,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/081e88d4ab274aeb85f65f6f8afa50d0, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:52,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/081e88d4ab274aeb85f65f6f8afa50d0 is 175, key is test_row_0/A:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:52,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742397_1573 (size=39949) 2024-12-05T11:16:52,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397472238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:52,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397472249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:52,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/081e88d4ab274aeb85f65f6f8afa50d0 2024-12-05T11:16:52,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/3713f9965d6a45d7b3e344d846262304 is 50, key is test_row_0/B:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:52,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742398_1574 (size=12301) 2024-12-05T11:16:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T11:16:52,669 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:52,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T11:16:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-05T11:16:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:16:52,671 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T11:16:52,671 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T11:16:52,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T11:16:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397472748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397472759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:16:52,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:16:52,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:52,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:52,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:52,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:52,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/3713f9965d6a45d7b3e344d846262304 2024-12-05T11:16:52,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/6d55efe696c542e1ba3f5826c6b20a56 is 50, key is test_row_0/C:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:52,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742399_1575 (size=12301) 2024-12-05T11:16:52,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:16:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:52,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58216 deadline: 1733397472980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:52,981 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6706a97ae65f,42931,1733397243639 for 'test_row_1' in ed350aebe3c2ac2bb91697ce75c09c2f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18327 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T11:16:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:16:53,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:16:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T11:16:53,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/6d55efe696c542e1ba3f5826c6b20a56 2024-12-05T11:16:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/081e88d4ab274aeb85f65f6f8afa50d0 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0 2024-12-05T11:16:53,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0, entries=200, sequenceid=282, filesize=39.0 K 2024-12-05T11:16:53,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/3713f9965d6a45d7b3e344d846262304 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/3713f9965d6a45d7b3e344d846262304 2024-12-05T11:16:53,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/3713f9965d6a45d7b3e344d846262304, entries=150, sequenceid=282, filesize=12.0 K 2024-12-05T11:16:53,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/6d55efe696c542e1ba3f5826c6b20a56 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6d55efe696c542e1ba3f5826c6b20a56 2024-12-05T11:16:53,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6d55efe696c542e1ba3f5826c6b20a56, entries=150, sequenceid=282, filesize=12.0 K 2024-12-05T11:16:53,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for ed350aebe3c2ac2bb91697ce75c09c2f in 1653ms, sequenceid=282, compaction requested=true 2024-12-05T11:16:53,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:53,254 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:53,255 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:53,255 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:53,255 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:53,256 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,256 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eda73d122c1848debed0ea9c69137e16, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=100.4 K 2024-12-05T11:16:53,256 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,256 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eda73d122c1848debed0ea9c69137e16, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0] 2024-12-05T11:16:53,257 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda73d122c1848debed0ea9c69137e16, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397407600 2024-12-05T11:16:53,257 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:53,257 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:53,257 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077f1842fead4d37982a543734a80e08, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733397408260 2024-12-05T11:16:53,258 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,258 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c39e5daedfb44728b5feb7f3152fefbd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/384932f273d849ec8a00bc31153640c6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/3713f9965d6a45d7b3e344d846262304] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.3 K 2024-12-05T11:16:53,258 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081e88d4ab274aeb85f65f6f8afa50d0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733397410442 2024-12-05T11:16:53,258 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting c39e5daedfb44728b5feb7f3152fefbd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397407600 2024-12-05T11:16:53,258 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 384932f273d849ec8a00bc31153640c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733397408260 2024-12-05T11:16:53,258 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 3713f9965d6a45d7b3e344d846262304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733397410442 2024-12-05T11:16:53,265 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:53,266 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/fe2564c920f94288abd3693686a7e131 is 50, key is test_row_0/B:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:53,267 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:53,270 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c013fdd1eba14126aa716f8f96fa08a3_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:53,272 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c013fdd1eba14126aa716f8f96fa08a3_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:53,272 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c013fdd1eba14126aa716f8f96fa08a3_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:53,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42931 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T11:16:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,285 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T11:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:16:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:16:53,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051c5055e6a9494a55bc352f456eb1dd21_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397411615/Put/seqid=0 2024-12-05T11:16:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742400_1576 (size=13017) 2024-12-05T11:16:53,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742401_1577 (size=4469) 2024-12-05T11:16:53,317 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/fe2564c920f94288abd3693686a7e131 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/fe2564c920f94288abd3693686a7e131 2024-12-05T11:16:53,320 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#481 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:53,320 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/4ad189c471154f9890cf2c969580ba90 is 175, key is test_row_0/A:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:53,326 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into fe2564c920f94288abd3693686a7e131(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:53,326 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:53,326 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397413255; duration=0sec 2024-12-05T11:16:53,327 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:53,327 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:53,327 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:53,329 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:53,329 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:53,329 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:53,330 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5299caecce9146beb96912dac74cef88, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1906e1bae13a4661b3600973b88bc066, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6d55efe696c542e1ba3f5826c6b20a56] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.3 K 2024-12-05T11:16:53,330 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 5299caecce9146beb96912dac74cef88, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733397407600 2024-12-05T11:16:53,330 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1906e1bae13a4661b3600973b88bc066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733397408260 2024-12-05T11:16:53,331 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d55efe696c542e1ba3f5826c6b20a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733397410442 2024-12-05T11:16:53,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742402_1578 (size=12454) 2024-12-05T11:16:53,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:53,351 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051c5055e6a9494a55bc352f456eb1dd21_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c5055e6a9494a55bc352f456eb1dd21_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/34d695e9f4934cc6a57fa9248778b68d, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:53,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/34d695e9f4934cc6a57fa9248778b68d is 175, key is test_row_0/A:col10/1733397411615/Put/seqid=0 2024-12-05T11:16:53,361 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:53,361 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1220c07b39324cc5944362b3feb3889b is 50, key is test_row_0/C:col10/1733397411600/Put/seqid=0 2024-12-05T11:16:53,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742403_1579 (size=31971) 2024-12-05T11:16:53,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742404_1580 (size=31255) 2024-12-05T11:16:53,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742405_1581 (size=13017) 2024-12-05T11:16:53,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. as already flushing 2024-12-05T11:16:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:53,782 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/4ad189c471154f9890cf2c969580ba90 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/4ad189c471154f9890cf2c969580ba90 2024-12-05T11:16:53,786 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into 4ad189c471154f9890cf2c969580ba90(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:53,787 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:53,787 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397413254; duration=0sec 2024-12-05T11:16:53,787 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:53,787 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:16:53,790 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/34d695e9f4934cc6a57fa9248778b68d 2024-12-05T11:16:53,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/b66dc2003ff74408b2566aa9bc1a9461 is 50, key is test_row_0/B:col10/1733397411615/Put/seqid=0 2024-12-05T11:16:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742406_1582 (size=12301) 2024-12-05T11:16:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:16:53,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397413808, max timestamp=1733397413817Num uniques 859; 2024-12-05T11:16:53,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:53,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397413808, max timestamp=1733397413817Num uniques 859; 2024-12-05T11:16:53,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:53,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733397413808, max timestamp=1733397413817Num uniques 859; 2024-12-05T11:16:53,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6706a97ae65f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T11:16:53,824 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/1220c07b39324cc5944362b3feb3889b as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1220c07b39324cc5944362b3feb3889b 2024-12-05T11:16:53,839 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into 1220c07b39324cc5944362b3feb3889b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:53,839 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:53,839 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397413255; duration=0sec 2024-12-05T11:16:53,839 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:53,839 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:53,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397473859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397473861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397473968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397473968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397474178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397474179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:54,202 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/b66dc2003ff74408b2566aa9bc1a9461 2024-12-05T11:16:54,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/dea97454452149b58fec346345118079 is 50, key is test_row_0/C:col10/1733397411615/Put/seqid=0 2024-12-05T11:16:54,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742407_1583 (size=12301) 2024-12-05T11:16:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T11:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1733397474489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42931 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58190 deadline: 1733397474489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 2024-12-05T11:16:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 , the old value is region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T11:16:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., hostname=6706a97ae65f,42931,1733397243639, seqNum=5 because the exception is null or not the one we care about 2024-12-05T11:16:54,593 INFO [Thread-2227 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2227. 2024-12-05T11:16:54,593 DEBUG [Thread-2227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,593 DEBUG [Thread-2227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,593 DEBUG [Thread-2227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,594 INFO [Thread-2219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2219. 2024-12-05T11:16:54,594 INFO [Thread-2221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2221. 2024-12-05T11:16:54,594 DEBUG [Thread-2219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,594 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,594 DEBUG [Thread-2221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,594 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,594 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,594 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,597 INFO [Thread-2225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2225. 2024-12-05T11:16:54,597 DEBUG [Thread-2225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,597 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,597 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,599 INFO [Thread-2223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2223. 2024-12-05T11:16:54,599 DEBUG [Thread-2223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,599 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,599 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,614 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/dea97454452149b58fec346345118079 2024-12-05T11:16:54,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/34d695e9f4934cc6a57fa9248778b68d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d 2024-12-05T11:16:54,620 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d, entries=150, sequenceid=292, filesize=30.5 K 2024-12-05T11:16:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/b66dc2003ff74408b2566aa9bc1a9461 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b66dc2003ff74408b2566aa9bc1a9461 2024-12-05T11:16:54,623 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b66dc2003ff74408b2566aa9bc1a9461, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:16:54,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/dea97454452149b58fec346345118079 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/dea97454452149b58fec346345118079 2024-12-05T11:16:54,625 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/dea97454452149b58fec346345118079, entries=150, sequenceid=292, filesize=12.0 K 2024-12-05T11:16:54,626 INFO [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ed350aebe3c2ac2bb91697ce75c09c2f in 1341ms, sequenceid=292, compaction requested=false 2024-12-05T11:16:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6706a97ae65f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-05T11:16:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-05T11:16:54,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-05T11:16:54,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9550 sec 2024-12-05T11:16:54,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.9600 sec 2024-12-05T11:16:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42931 {}] regionserver.HRegion(8855): Flush requested on ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:54,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-05T11:16:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:16:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:16:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:16:54,700 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-12-05T11:16:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:16:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:16:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T11:16:54,700 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,700 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,700 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ff75455d8c7e4ad18dbb600525036d9b_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_0/A:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:54,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742408_1584 (size=12454) 2024-12-05T11:16:54,720 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-12-05T11:16:54,720 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:54,720 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,720 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:54,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T11:16:54,819 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:16:55,000 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-12-05T11:16:55,000 INFO [Thread-2208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2208. 2024-12-05T11:16:55,000 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:55,000 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:55,000 DEBUG [Thread-2208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:16:55,000 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:55,000 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:55,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:55,000 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:16:55,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:16:55,108 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:16:55,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ff75455d8c7e4ad18dbb600525036d9b_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff75455d8c7e4ad18dbb600525036d9b_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:16:55,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/0862ee71a1dc4a7794f9949546a327e9, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:55,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/0862ee71a1dc4a7794f9949546a327e9 is 175, key is test_row_0/A:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742409_1585 (size=31255) 2024-12-05T11:16:55,515 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/0862ee71a1dc4a7794f9949546a327e9 2024-12-05T11:16:55,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/14dc57e199f143cabd43fc596b483e89 is 50, key is test_row_0/B:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:55,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742410_1586 (size=12301) 2024-12-05T11:16:55,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/14dc57e199f143cabd43fc596b483e89 2024-12-05T11:16:55,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/cbf9348005f24547ba5567444f46978e is 50, key is test_row_0/C:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742411_1587 (size=12301) 2024-12-05T11:16:56,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/cbf9348005f24547ba5567444f46978e 2024-12-05T11:16:56,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/0862ee71a1dc4a7794f9949546a327e9 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9 2024-12-05T11:16:56,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9, entries=150, sequenceid=333, filesize=30.5 K 2024-12-05T11:16:56,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/14dc57e199f143cabd43fc596b483e89 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/14dc57e199f143cabd43fc596b483e89 2024-12-05T11:16:56,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/14dc57e199f143cabd43fc596b483e89, entries=150, sequenceid=333, filesize=12.0 K 2024-12-05T11:16:56,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/cbf9348005f24547ba5567444f46978e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/cbf9348005f24547ba5567444f46978e 2024-12-05T11:16:56,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/cbf9348005f24547ba5567444f46978e, entries=150, sequenceid=333, filesize=12.0 K 2024-12-05T11:16:56,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=20.13 KB/20610 for ed350aebe3c2ac2bb91697ce75c09c2f in 1646ms, sequenceid=333, compaction requested=true 2024-12-05T11:16:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T11:16:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T11:16:56,346 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:56,346 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed350aebe3c2ac2bb91697ce75c09c2f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T11:16:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:56,346 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:56,346 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:56,346 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/B is initiating minor compaction (all files) 2024-12-05T11:16:56,346 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/A is initiating minor compaction (all files) 2024-12-05T11:16:56,346 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/B in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:56,347 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/A in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:56,347 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/fe2564c920f94288abd3693686a7e131, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b66dc2003ff74408b2566aa9bc1a9461, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/14dc57e199f143cabd43fc596b483e89] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.7 K 2024-12-05T11:16:56,347 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/4ad189c471154f9890cf2c969580ba90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=92.3 K 2024-12-05T11:16:56,347 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. files: [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/4ad189c471154f9890cf2c969580ba90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9] 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2564c920f94288abd3693686a7e131, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733397410442 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad189c471154f9890cf2c969580ba90, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733397410442 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting b66dc2003ff74408b2566aa9bc1a9461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397411608 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34d695e9f4934cc6a57fa9248778b68d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397411608 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 14dc57e199f143cabd43fc596b483e89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733397413858 2024-12-05T11:16:56,347 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0862ee71a1dc4a7794f9949546a327e9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733397413858 2024-12-05T11:16:56,352 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:56,352 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:56,353 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/4354f236b52242ac9e667b4cba32933a is 50, key is test_row_0/B:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:56,353 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205a5d880e213264c568693f2e97c94ec41_ed350aebe3c2ac2bb91697ce75c09c2f store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:56,356 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205a5d880e213264c568693f2e97c94ec41_ed350aebe3c2ac2bb91697ce75c09c2f, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:56,356 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a5d880e213264c568693f2e97c94ec41_ed350aebe3c2ac2bb91697ce75c09c2f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:16:56,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742412_1588 (size=13119) 2024-12-05T11:16:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742413_1589 (size=4469) 2024-12-05T11:16:56,759 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#A#compaction#490 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:56,760 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/a82a9f6020554b76b6f78fd6a4e073c5 is 175, key is test_row_0/A:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:56,762 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/4354f236b52242ac9e667b4cba32933a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/4354f236b52242ac9e667b4cba32933a 2024-12-05T11:16:56,767 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/B of ed350aebe3c2ac2bb91697ce75c09c2f into 4354f236b52242ac9e667b4cba32933a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:56,767 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:56,767 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/B, priority=13, startTime=1733397416346; duration=0sec 2024-12-05T11:16:56,767 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T11:16:56,767 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:B 2024-12-05T11:16:56,767 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T11:16:56,768 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T11:16:56,768 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1541): ed350aebe3c2ac2bb91697ce75c09c2f/C is initiating minor compaction (all files) 2024-12-05T11:16:56,769 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed350aebe3c2ac2bb91697ce75c09c2f/C in TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:16:56,769 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1220c07b39324cc5944362b3feb3889b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/dea97454452149b58fec346345118079, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/cbf9348005f24547ba5567444f46978e] into tmpdir=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp, totalSize=36.7 K 2024-12-05T11:16:56,769 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting 1220c07b39324cc5944362b3feb3889b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733397410442 2024-12-05T11:16:56,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742414_1590 (size=32073) 2024-12-05T11:16:56,770 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting dea97454452149b58fec346345118079, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733397411608 2024-12-05T11:16:56,770 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] compactions.Compactor(225): Compacting cbf9348005f24547ba5567444f46978e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733397413858 2024-12-05T11:16:56,775 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed350aebe3c2ac2bb91697ce75c09c2f#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T11:16:56,775 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/490551d200414d1db65dbe749c3d369d is 50, key is test_row_0/C:col10/1733397414698/Put/seqid=0 2024-12-05T11:16:56,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742415_1591 (size=13119) 2024-12-05T11:16:56,785 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/490551d200414d1db65dbe749c3d369d as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/490551d200414d1db65dbe749c3d369d 2024-12-05T11:16:56,788 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/C of ed350aebe3c2ac2bb91697ce75c09c2f into 490551d200414d1db65dbe749c3d369d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:56,788 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:56,788 INFO [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/C, priority=13, startTime=1733397416346; duration=0sec 2024-12-05T11:16:56,788 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:56,788 DEBUG [RS:0;6706a97ae65f:42931-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:C 2024-12-05T11:16:57,177 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/a82a9f6020554b76b6f78fd6a4e073c5 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/a82a9f6020554b76b6f78fd6a4e073c5 2024-12-05T11:16:57,183 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed350aebe3c2ac2bb91697ce75c09c2f/A of ed350aebe3c2ac2bb91697ce75c09c2f into a82a9f6020554b76b6f78fd6a4e073c5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T11:16:57,183 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed350aebe3c2ac2bb91697ce75c09c2f: 2024-12-05T11:16:57,183 INFO [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., storeName=ed350aebe3c2ac2bb91697ce75c09c2f/A, priority=13, startTime=1733397416345; duration=0sec 2024-12-05T11:16:57,183 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T11:16:57,183 DEBUG [RS:0;6706a97ae65f:42931-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed350aebe3c2ac2bb91697ce75c09c2f:A 2024-12-05T11:17:02,094 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T11:17:03,063 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-12-05T11:17:03,064 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T11:17:03,064 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:03,064 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:03,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:17:03,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4666 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4628 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4539 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4665 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4630 2024-12-05T11:17:03,065 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T11:17:03,065 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:17:03,065 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:17:03,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:03,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:03,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:17:03,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T11:17:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:17:03,070 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397423070"}]},"ts":"1733397423070"} 2024-12-05T11:17:03,072 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T11:17:03,072 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T11:17:03,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T11:17:03,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, UNASSIGN}] 2024-12-05T11:17:03,075 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, UNASSIGN 2024-12-05T11:17:03,075 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=CLOSING, regionLocation=6706a97ae65f,42931,1733397243639 2024-12-05T11:17:03,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, UNASSIGN because future has completed 2024-12-05T11:17:03,077 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T11:17:03,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639}] 2024-12-05T11:17:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:17:03,229 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(122): Close ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:03,229 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T11:17:03,229 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1722): Closing ed350aebe3c2ac2bb91697ce75c09c2f, disabling compactions & flushes 2024-12-05T11:17:03,229 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:17:03,229 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:17:03,229 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. after waiting 0 ms 2024-12-05T11:17:03,229 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:17:03,230 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(2902): Flushing ed350aebe3c2ac2bb91697ce75c09c2f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T11:17:03,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=A 2024-12-05T11:17:03,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:17:03,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=B 2024-12-05T11:17:03,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:17:03,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed350aebe3c2ac2bb91697ce75c09c2f, store=C 2024-12-05T11:17:03,230 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T11:17:03,256 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057e8a4458f5d24f808e67676d0299e525_ed350aebe3c2ac2bb91697ce75c09c2f is 50, key is test_row_1/A:col10/1733397423060/Put/seqid=0 2024-12-05T11:17:03,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742416_1592 (size=9914) 2024-12-05T11:17:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:17:03,660 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T11:17:03,663 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057e8a4458f5d24f808e67676d0299e525_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057e8a4458f5d24f808e67676d0299e525_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:03,664 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6fcf3cb892d842b3810760bf1296782a, store: [table=TestAcidGuarantees family=A region=ed350aebe3c2ac2bb91697ce75c09c2f] 2024-12-05T11:17:03,664 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6fcf3cb892d842b3810760bf1296782a is 175, key is test_row_1/A:col10/1733397423060/Put/seqid=0 2024-12-05T11:17:03,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742417_1593 (size=22561) 2024-12-05T11:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:17:04,069 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6fcf3cb892d842b3810760bf1296782a 2024-12-05T11:17:04,080 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8559d1260e18488aaf04a85a03f9b454 is 50, key is test_row_1/B:col10/1733397423060/Put/seqid=0 2024-12-05T11:17:04,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742418_1594 (size=9857) 2024-12-05T11:17:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:17:04,484 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8559d1260e18488aaf04a85a03f9b454 2024-12-05T11:17:04,489 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/013662219027432e881e8ac2bd802e55 is 50, key is test_row_1/C:col10/1733397423060/Put/seqid=0 2024-12-05T11:17:04,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742419_1595 (size=9857) 2024-12-05T11:17:04,893 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/013662219027432e881e8ac2bd802e55 2024-12-05T11:17:04,898 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/A/6fcf3cb892d842b3810760bf1296782a as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6fcf3cb892d842b3810760bf1296782a 2024-12-05T11:17:04,904 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6fcf3cb892d842b3810760bf1296782a, entries=100, sequenceid=343, filesize=22.0 K 2024-12-05T11:17:04,905 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/B/8559d1260e18488aaf04a85a03f9b454 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8559d1260e18488aaf04a85a03f9b454 2024-12-05T11:17:04,909 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8559d1260e18488aaf04a85a03f9b454, entries=100, sequenceid=343, filesize=9.6 K 2024-12-05T11:17:04,910 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/.tmp/C/013662219027432e881e8ac2bd802e55 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/013662219027432e881e8ac2bd802e55 2024-12-05T11:17:04,913 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/013662219027432e881e8ac2bd802e55, entries=100, sequenceid=343, filesize=9.6 K 2024-12-05T11:17:04,914 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ed350aebe3c2ac2bb91697ce75c09c2f in 1685ms, sequenceid=343, compaction requested=false 2024-12-05T11:17:04,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f090d99765744dadb98d42595abe0b53, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90a52057dc084989b6be8d0110f02e09, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/15244aa119044a16a14bcd9bb1eed8aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/bf510443f65441bba4f7e597384dd7aa, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f90a5a746708428493a46953a1ec23f9, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eda73d122c1848debed0ea9c69137e16, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/4ad189c471154f9890cf2c969580ba90, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9] to archive 2024-12-05T11:17:04,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:17:04,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/7f36fe4270cf412bbd1c140ce2553161 2024-12-05T11:17:04,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6596a2435aca4798b53b400599ab2a32 2024-12-05T11:17:04,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d75927a4c47f421db0b18d746505c2b3 2024-12-05T11:17:04,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f090d99765744dadb98d42595abe0b53 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f090d99765744dadb98d42595abe0b53 2024-12-05T11:17:04,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eaa1119c390d4454bdba976f1d141aa7 2024-12-05T11:17:04,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/c8675dc89737482182832daab27b7b94 2024-12-05T11:17:04,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90a52057dc084989b6be8d0110f02e09 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90a52057dc084989b6be8d0110f02e09 2024-12-05T11:17:04,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/90294bf747d6443e858eae4a916c1b30 2024-12-05T11:17:04,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/55a48973c5c241b38d68c62b4fb5081c 2024-12-05T11:17:04,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/15244aa119044a16a14bcd9bb1eed8aa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/15244aa119044a16a14bcd9bb1eed8aa 2024-12-05T11:17:04,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/b933af4fbdf8485b9b148a5d4ed02bad 2024-12-05T11:17:04,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/1763b10dd5874cd4bc3239b678901ec4 2024-12-05T11:17:04,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/bf510443f65441bba4f7e597384dd7aa to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/bf510443f65441bba4f7e597384dd7aa 2024-12-05T11:17:04,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/88d014dc2a0c4ee8aede9b8c8a411548 2024-12-05T11:17:04,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/2bd1e85993134d3a9a83da84a0446232 2024-12-05T11:17:04,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f90a5a746708428493a46953a1ec23f9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/f90a5a746708428493a46953a1ec23f9 2024-12-05T11:17:04,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d401cd9df10a464babc498383b436554 2024-12-05T11:17:04,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/d4cbe082889d43fa86fd3c9b7de3fb05 2024-12-05T11:17:04,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eda73d122c1848debed0ea9c69137e16 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/eda73d122c1848debed0ea9c69137e16 2024-12-05T11:17:04,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/077f1842fead4d37982a543734a80e08 2024-12-05T11:17:04,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/081e88d4ab274aeb85f65f6f8afa50d0 2024-12-05T11:17:04,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/4ad189c471154f9890cf2c969580ba90 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/4ad189c471154f9890cf2c969580ba90 2024-12-05T11:17:04,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/34d695e9f4934cc6a57fa9248778b68d 2024-12-05T11:17:04,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/0862ee71a1dc4a7794f9949546a327e9 2024-12-05T11:17:04,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/7239fa4abfb2417c9cd637faea39f8c0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/22b830811176442d86efef727cb74716, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c5b77ab17cc64390b2da64d165549bf1, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8700a85e864b4571ab8d2001abf3ac5d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/ded685274d3d4e4997f1aeceafba5064, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/e6c291fbd12e421c8aa1e179e5b948a0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/dacf38ecc4fd46c69a96362712455e94, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/516a5c6e6ad64061aa1b0099bf181bd6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6b11aa1d2084493389d128a70b21a9df, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/1d37617b83544961ba810d0aeece1873, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/40204c71536a48a8bebdf748600c88ee, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c411bad288154dee819b32f47ba32089, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/f957767d7f044eb6b4901e2dd460dbca, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/af1c6cf6614a4f1fa2ad5764a9311558, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b7d771db7ab441399c2597fd25dfaf70, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/cec09162effd4f03b2417c751592a4df, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6201db35fcfe4e9faf1a43989fed4c0d, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c39e5daedfb44728b5feb7f3152fefbd, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8af4344abb9142d1a7fdc027bc97b09c, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/384932f273d849ec8a00bc31153640c6, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/fe2564c920f94288abd3693686a7e131, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/3713f9965d6a45d7b3e344d846262304, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b66dc2003ff74408b2566aa9bc1a9461, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/14dc57e199f143cabd43fc596b483e89] to archive 2024-12-05T11:17:04,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:17:04,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/7239fa4abfb2417c9cd637faea39f8c0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/7239fa4abfb2417c9cd637faea39f8c0 2024-12-05T11:17:04,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/22b830811176442d86efef727cb74716 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/22b830811176442d86efef727cb74716 2024-12-05T11:17:04,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c5b77ab17cc64390b2da64d165549bf1 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c5b77ab17cc64390b2da64d165549bf1 2024-12-05T11:17:04,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8700a85e864b4571ab8d2001abf3ac5d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8700a85e864b4571ab8d2001abf3ac5d 2024-12-05T11:17:04,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/ded685274d3d4e4997f1aeceafba5064 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/ded685274d3d4e4997f1aeceafba5064 2024-12-05T11:17:04,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/e6c291fbd12e421c8aa1e179e5b948a0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/e6c291fbd12e421c8aa1e179e5b948a0 2024-12-05T11:17:04,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/dacf38ecc4fd46c69a96362712455e94 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/dacf38ecc4fd46c69a96362712455e94 2024-12-05T11:17:04,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/516a5c6e6ad64061aa1b0099bf181bd6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/516a5c6e6ad64061aa1b0099bf181bd6 2024-12-05T11:17:04,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6b11aa1d2084493389d128a70b21a9df to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6b11aa1d2084493389d128a70b21a9df 2024-12-05T11:17:04,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/1d37617b83544961ba810d0aeece1873 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/1d37617b83544961ba810d0aeece1873 2024-12-05T11:17:04,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/40204c71536a48a8bebdf748600c88ee to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/40204c71536a48a8bebdf748600c88ee 2024-12-05T11:17:04,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c411bad288154dee819b32f47ba32089 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c411bad288154dee819b32f47ba32089 2024-12-05T11:17:04,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/f957767d7f044eb6b4901e2dd460dbca to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/f957767d7f044eb6b4901e2dd460dbca 2024-12-05T11:17:04,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/af1c6cf6614a4f1fa2ad5764a9311558 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/af1c6cf6614a4f1fa2ad5764a9311558 2024-12-05T11:17:04,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b7d771db7ab441399c2597fd25dfaf70 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b7d771db7ab441399c2597fd25dfaf70 2024-12-05T11:17:04,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/cec09162effd4f03b2417c751592a4df to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/cec09162effd4f03b2417c751592a4df 2024-12-05T11:17:04,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6201db35fcfe4e9faf1a43989fed4c0d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/6201db35fcfe4e9faf1a43989fed4c0d 2024-12-05T11:17:04,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c39e5daedfb44728b5feb7f3152fefbd to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/c39e5daedfb44728b5feb7f3152fefbd 2024-12-05T11:17:04,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8af4344abb9142d1a7fdc027bc97b09c to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8af4344abb9142d1a7fdc027bc97b09c 2024-12-05T11:17:04,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/384932f273d849ec8a00bc31153640c6 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/384932f273d849ec8a00bc31153640c6 2024-12-05T11:17:04,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/fe2564c920f94288abd3693686a7e131 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/fe2564c920f94288abd3693686a7e131 2024-12-05T11:17:04,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/3713f9965d6a45d7b3e344d846262304 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/3713f9965d6a45d7b3e344d846262304 2024-12-05T11:17:04,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b66dc2003ff74408b2566aa9bc1a9461 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/b66dc2003ff74408b2566aa9bc1a9461 2024-12-05T11:17:04,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/14dc57e199f143cabd43fc596b483e89 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/14dc57e199f143cabd43fc596b483e89 2024-12-05T11:17:04,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0d2fdc67cc194964acb05128f5154bcc, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/111cd5d57c6b4d87bfe9ecc88cba4531, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0dcad02b0db74e6e9c97f30c8d033766, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/3dece7e699c6426296242f18489e6d26, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/72b93a58067e42be9e700b447cb75ff2, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c7f25f3c77c34c9682b18c1d61d8e0a0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/25640ecd99914c38b46d1cd864d4b817, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/361e4ed4a7fc4da5a7411a30cf947528, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/f8c6d7956f604263bd61ec06d951bdf0, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ee26211990a84cebbb2cd21dae47fbce, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1eccbfa91282496b98b707bbe8a0b882, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/b1b35826144340c38169b20227c56dc7, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/d11178f0712b4c7e9f63a94cc0e9dde3, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5898f0ebd94b484781e91d985c0fc1b4, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ce8a9c9b55a64498b370f379e72e1595, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6f685fdc55354581bedb1c856068dfeb, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c3eece5250e1458cae86443d607f8a66, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5299caecce9146beb96912dac74cef88, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/462c1adcba22400bb6e41eb5405b3425, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1906e1bae13a4661b3600973b88bc066, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1220c07b39324cc5944362b3feb3889b, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6d55efe696c542e1ba3f5826c6b20a56, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/dea97454452149b58fec346345118079, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/cbf9348005f24547ba5567444f46978e] to archive 2024-12-05T11:17:04,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T11:17:04,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0d2fdc67cc194964acb05128f5154bcc to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0d2fdc67cc194964acb05128f5154bcc 2024-12-05T11:17:04,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/111cd5d57c6b4d87bfe9ecc88cba4531 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/111cd5d57c6b4d87bfe9ecc88cba4531 2024-12-05T11:17:04,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0dcad02b0db74e6e9c97f30c8d033766 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/0dcad02b0db74e6e9c97f30c8d033766 2024-12-05T11:17:04,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/3dece7e699c6426296242f18489e6d26 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/3dece7e699c6426296242f18489e6d26 2024-12-05T11:17:04,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/72b93a58067e42be9e700b447cb75ff2 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/72b93a58067e42be9e700b447cb75ff2 2024-12-05T11:17:04,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c7f25f3c77c34c9682b18c1d61d8e0a0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c7f25f3c77c34c9682b18c1d61d8e0a0 2024-12-05T11:17:04,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/25640ecd99914c38b46d1cd864d4b817 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/25640ecd99914c38b46d1cd864d4b817 2024-12-05T11:17:04,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/361e4ed4a7fc4da5a7411a30cf947528 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/361e4ed4a7fc4da5a7411a30cf947528 2024-12-05T11:17:04,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/f8c6d7956f604263bd61ec06d951bdf0 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/f8c6d7956f604263bd61ec06d951bdf0 2024-12-05T11:17:04,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ee26211990a84cebbb2cd21dae47fbce to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ee26211990a84cebbb2cd21dae47fbce 2024-12-05T11:17:04,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1eccbfa91282496b98b707bbe8a0b882 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1eccbfa91282496b98b707bbe8a0b882 2024-12-05T11:17:04,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/b1b35826144340c38169b20227c56dc7 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/b1b35826144340c38169b20227c56dc7 2024-12-05T11:17:04,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/d11178f0712b4c7e9f63a94cc0e9dde3 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/d11178f0712b4c7e9f63a94cc0e9dde3 2024-12-05T11:17:04,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5898f0ebd94b484781e91d985c0fc1b4 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5898f0ebd94b484781e91d985c0fc1b4 2024-12-05T11:17:04,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ce8a9c9b55a64498b370f379e72e1595 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/ce8a9c9b55a64498b370f379e72e1595 2024-12-05T11:17:04,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6f685fdc55354581bedb1c856068dfeb to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6f685fdc55354581bedb1c856068dfeb 2024-12-05T11:17:04,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c3eece5250e1458cae86443d607f8a66 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/c3eece5250e1458cae86443d607f8a66 2024-12-05T11:17:04,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5299caecce9146beb96912dac74cef88 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/5299caecce9146beb96912dac74cef88 2024-12-05T11:17:04,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/462c1adcba22400bb6e41eb5405b3425 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/462c1adcba22400bb6e41eb5405b3425 2024-12-05T11:17:04,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1906e1bae13a4661b3600973b88bc066 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1906e1bae13a4661b3600973b88bc066 2024-12-05T11:17:04,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1220c07b39324cc5944362b3feb3889b to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/1220c07b39324cc5944362b3feb3889b 2024-12-05T11:17:04,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6d55efe696c542e1ba3f5826c6b20a56 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/6d55efe696c542e1ba3f5826c6b20a56 2024-12-05T11:17:04,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/dea97454452149b58fec346345118079 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/dea97454452149b58fec346345118079 2024-12-05T11:17:04,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/cbf9348005f24547ba5567444f46978e to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/cbf9348005f24547ba5567444f46978e 2024-12-05T11:17:05,001 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=4 2024-12-05T11:17:05,002 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. 2024-12-05T11:17:05,002 DEBUG [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1676): Region close journal for ed350aebe3c2ac2bb91697ce75c09c2f: Waiting for close lock at 1733397423229Running coprocessor pre-close hooks at 1733397423229Disabling compacts and flushes for region at 1733397423229Disabling writes for close at 1733397423229Obtaining lock to block concurrent updates at 1733397423230 (+1 ms)Preparing flush snapshotting stores in ed350aebe3c2ac2bb91697ce75c09c2f at 1733397423230Finished memstore snapshotting TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733397423230Flushing stores of TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f. at 1733397423251 (+21 ms)Flushing ed350aebe3c2ac2bb91697ce75c09c2f/A: creating writer at 1733397423252 (+1 ms)Flushing mob file ed350aebe3c2ac2bb91697ce75c09c2f/A: appending metadata at 1733397423256 (+4 ms)Flushing mob file ed350aebe3c2ac2bb91697ce75c09c2f/A: closing flushed file at 1733397423256Flushing ed350aebe3c2ac2bb91697ce75c09c2f/A: appending metadata at 1733397423664 (+408 ms)Flushing ed350aebe3c2ac2bb91697ce75c09c2f/A: closing flushed file at 1733397423664Flushing ed350aebe3c2ac2bb91697ce75c09c2f/B: creating writer at 1733397424076 (+412 ms)Flushing ed350aebe3c2ac2bb91697ce75c09c2f/B: appending metadata at 1733397424079 (+3 ms)Flushing ed350aebe3c2ac2bb91697ce75c09c2f/B: closing flushed file at 1733397424079Flushing ed350aebe3c2ac2bb91697ce75c09c2f/C: creating writer at 1733397424487 (+408 ms)Flushing ed350aebe3c2ac2bb91697ce75c09c2f/C: appending metadata at 1733397424489 (+2 ms)Flushing ed350aebe3c2ac2bb91697ce75c09c2f/C: closing flushed file at 1733397424489Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b76fcbe: reopening flushed file at 1733397424897 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734dbaa6: reopening flushed file at 1733397424904 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a420ef0: reopening flushed file at 1733397424909 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ed350aebe3c2ac2bb91697ce75c09c2f in 1685ms, sequenceid=343, compaction requested=false at 1733397424914 (+5 ms)Writing region close event to WAL at 1733397424999 (+85 ms)Running coprocessor post-close hooks at 1733397425002 (+3 ms)Closed at 1733397425002 2024-12-05T11:17:05,003 INFO [RS_CLOSE_REGION-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(157): Closed ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=ed350aebe3c2ac2bb91697ce75c09c2f, regionState=CLOSED 2024-12-05T11:17:05,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 because future has completed 2024-12-05T11:17:05,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-05T11:17:05,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseRegionProcedure ed350aebe3c2ac2bb91697ce75c09c2f, server=6706a97ae65f,42931,1733397243639 in 1.9290 sec 2024-12-05T11:17:05,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-12-05T11:17:05,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed350aebe3c2ac2bb91697ce75c09c2f, UNASSIGN in 1.9330 sec 2024-12-05T11:17:05,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-05T11:17:05,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9370 sec 2024-12-05T11:17:05,011 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733397425011"}]},"ts":"1733397425011"} 2024-12-05T11:17:05,012 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T11:17:05,012 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T11:17:05,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9450 sec 2024-12-05T11:17:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T11:17:05,219 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:17:05,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T11:17:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:05,221 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:17:05,222 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=158, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:05,223 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,225 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C, FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/recovered.edits] 2024-12-05T11:17:05,227 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6fcf3cb892d842b3810760bf1296782a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/6fcf3cb892d842b3810760bf1296782a 2024-12-05T11:17:05,228 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/a82a9f6020554b76b6f78fd6a4e073c5 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/A/a82a9f6020554b76b6f78fd6a4e073c5 2024-12-05T11:17:05,230 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/4354f236b52242ac9e667b4cba32933a to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/4354f236b52242ac9e667b4cba32933a 2024-12-05T11:17:05,231 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8559d1260e18488aaf04a85a03f9b454 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/B/8559d1260e18488aaf04a85a03f9b454 2024-12-05T11:17:05,233 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/013662219027432e881e8ac2bd802e55 to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/013662219027432e881e8ac2bd802e55 2024-12-05T11:17:05,234 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/490551d200414d1db65dbe749c3d369d to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/C/490551d200414d1db65dbe749c3d369d 2024-12-05T11:17:05,236 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/recovered.edits/346.seqid to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f/recovered.edits/346.seqid 2024-12-05T11:17:05,236 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/default/TestAcidGuarantees/ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,237 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T11:17:05,237 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:17:05,237 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T11:17:05,239 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050048fc00c9ea42fbacd18861aa88747a_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050048fc00c9ea42fbacd18861aa88747a_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,240 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120503884e0c1e3b4663be0637667c1a00b1_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120503884e0c1e3b4663be0637667c1a00b1_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,240 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c5055e6a9494a55bc352f456eb1dd21_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051c5055e6a9494a55bc352f456eb1dd21_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,241 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205249bf2a6364c4d80ba1f1f3aad30b1c6_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205249bf2a6364c4d80ba1f1f3aad30b1c6_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,242 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c4a2ade273949d6920ac941e36f2400_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053c4a2ade273949d6920ac941e36f2400_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,242 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053e97489dbff84555aad8555949d62865_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053e97489dbff84555aad8555949d62865_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,243 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054efeedbee3ca4964a46ee763d8d5ec90_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054efeedbee3ca4964a46ee763d8d5ec90_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,243 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057e2fd6011c294c8188743a4b2d981bd5_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057e2fd6011c294c8188743a4b2d981bd5_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,244 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057e8a4458f5d24f808e67676d0299e525_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057e8a4458f5d24f808e67676d0299e525_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,245 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058277e722529d49feb8ddaa9fa55f4601_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412058277e722529d49feb8ddaa9fa55f4601_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,245 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205938251cfc4354b5cb5d72af43fb1f4ef_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205938251cfc4354b5cb5d72af43fb1f4ef_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,246 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059a87b9e5b0d94a76abf7d73dcb07d75f_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059a87b9e5b0d94a76abf7d73dcb07d75f_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,246 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c62d400b80374fa188a785cd84bcdc9f_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c62d400b80374fa188a785cd84bcdc9f_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,247 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d02ed50d3a1244aaa3310f2467f86f96_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d02ed50d3a1244aaa3310f2467f86f96_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,248 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205edead28e4de14e498f61da0690faf59e_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205edead28e4de14e498f61da0690faf59e_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,249 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f5b44fd12fc244ef8cfc7fc24d04aa5c_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f5b44fd12fc244ef8cfc7fc24d04aa5c_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,249 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fbcdc1e3a34e4fd0904223eb7894d4ce_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fbcdc1e3a34e4fd0904223eb7894d4ce_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,250 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff75455d8c7e4ad18dbb600525036d9b_ed350aebe3c2ac2bb91697ce75c09c2f to hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff75455d8c7e4ad18dbb600525036d9b_ed350aebe3c2ac2bb91697ce75c09c2f 2024-12-05T11:17:05,251 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T11:17:05,252 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=158, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:05,254 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T11:17:05,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T11:17:05,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=158, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:05,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T11:17:05,257 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733397425257"}]},"ts":"9223372036854775807"} 2024-12-05T11:17:05,259 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T11:17:05,259 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ed350aebe3c2ac2bb91697ce75c09c2f, NAME => 'TestAcidGuarantees,,1733397391394.ed350aebe3c2ac2bb91697ce75c09c2f.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T11:17:05,259 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T11:17:05,259 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733397425259"}]},"ts":"9223372036854775807"} 2024-12-05T11:17:05,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T11:17:05,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=158, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T11:17:05,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-05T11:17:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38049 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T11:17:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T11:17:05,329 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T11:17:05,339 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=428 (was 456), ProcessCount=11 (was 11), AvailableMemoryMB=6527 (was 5885) - AvailableMemoryMB LEAK? - 2024-12-05T11:17:05,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T11:17:05,339 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:17:05,339 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:17:05,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:05,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:05,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T11:17:05,340 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T11:17:05,340 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-12-05T11:17:05,340 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6706a97ae65f,38049,1733397242931 2024-12-05T11:17:05,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T11:17:05,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T11:17:05,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:17:05,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:17:05,363 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T11:17:05,364 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T11:17:05,364 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:17:05,364 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:17:05,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:05,365 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T11:17:05,365 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6706a97ae65f,42931,1733397243639' ***** 2024-12-05T11:17:05,366 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T11:17:05,366 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T11:17:05,366 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T11:17:05,367 INFO [RS:0;6706a97ae65f:42931 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T11:17:05,367 INFO [RS:0;6706a97ae65f:42931 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T11:17:05,367 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(959): stopping server 6706a97ae65f,42931,1733397243639 2024-12-05T11:17:05,367 INFO [RS:0;6706a97ae65f:42931 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T11:17:05,368 INFO [RS:0;6706a97ae65f:42931 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6706a97ae65f:42931. 2024-12-05T11:17:05,368 DEBUG [RS:0;6706a97ae65f:42931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T11:17:05,368 DEBUG [RS:0;6706a97ae65f:42931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:05,368 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T11:17:05,368 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T11:17:05,368 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T11:17:05,368 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T11:17:05,369 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T11:17:05,369 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T11:17:05,369 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:05,369 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T11:17:05,369 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T11:17:05,369 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T11:17:05,369 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T11:17:05,369 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T11:17:05,369 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-05T11:17:05,389 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/info/a0c2071aeb924bf4b6b8bad0c7252a01 is 95, key is TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9./info:/1733397275440/DeleteFamily/seqid=0 2024-12-05T11:17:05,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742420_1596 (size=6514) 2024-12-05T11:17:05,447 INFO [regionserver/6706a97ae65f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T11:17:05,569 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:05,758 INFO [regionserver/6706a97ae65f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T11:17:05,758 INFO [regionserver/6706a97ae65f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T11:17:05,770 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:05,792 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/info/a0c2071aeb924bf4b6b8bad0c7252a01 2024-12-05T11:17:05,809 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/ns/612b1aa3a3714502b1ae9e7db9215136 is 93, key is TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9./ns:/1733397275400/DeleteFamily/seqid=0 2024-12-05T11:17:05,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742421_1597 (size=5906) 2024-12-05T11:17:05,970 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:06,170 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:06,213 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/ns/612b1aa3a3714502b1ae9e7db9215136 2024-12-05T11:17:06,238 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/rep_barrier/b3ea99198287498480260eaac207a764 is 102, key is TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9./rep_barrier:/1733397275400/DeleteFamily/seqid=0 2024-12-05T11:17:06,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742422_1598 (size=6025) 2024-12-05T11:17:06,371 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:06,571 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:06,643 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/rep_barrier/b3ea99198287498480260eaac207a764 2024-12-05T11:17:06,671 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/table/b441a22bafc641099f8dc9aa9a719a96 is 96, key is TestAcidGuarantees,,1733397248326.fb2f3b651a249ac1bc6c3ae631d8f7b9./table:/1733397275400/DeleteFamily/seqid=0 2024-12-05T11:17:06,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742423_1599 (size=5936) 2024-12-05T11:17:06,771 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:06,771 INFO [regionserver/6706a97ae65f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-12-05T11:17:06,971 DEBUG [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T11:17:07,075 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/table/b441a22bafc641099f8dc9aa9a719a96 2024-12-05T11:17:07,083 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/info/a0c2071aeb924bf4b6b8bad0c7252a01 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/info/a0c2071aeb924bf4b6b8bad0c7252a01 2024-12-05T11:17:07,089 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/info/a0c2071aeb924bf4b6b8bad0c7252a01, entries=12, sequenceid=90, filesize=6.4 K 2024-12-05T11:17:07,090 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/ns/612b1aa3a3714502b1ae9e7db9215136 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/ns/612b1aa3a3714502b1ae9e7db9215136 2024-12-05T11:17:07,093 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/ns/612b1aa3a3714502b1ae9e7db9215136, entries=8, sequenceid=90, filesize=5.8 K 2024-12-05T11:17:07,094 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/rep_barrier/b3ea99198287498480260eaac207a764 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/rep_barrier/b3ea99198287498480260eaac207a764 2024-12-05T11:17:07,096 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/rep_barrier/b3ea99198287498480260eaac207a764, entries=6, sequenceid=90, filesize=5.9 K 2024-12-05T11:17:07,097 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/.tmp/table/b441a22bafc641099f8dc9aa9a719a96 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/table/b441a22bafc641099f8dc9aa9a719a96 2024-12-05T11:17:07,100 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/table/b441a22bafc641099f8dc9aa9a719a96, entries=7, sequenceid=90, filesize=5.8 K 2024-12-05T11:17:07,101 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1731ms, sequenceid=90, compaction requested=false 2024-12-05T11:17:07,104 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-05T11:17:07,105 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T11:17:07,105 INFO [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T11:17:07,105 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733397425369Running coprocessor pre-close hooks at 1733397425369Disabling compacts and flushes for region at 1733397425369Disabling writes for close at 1733397425369Obtaining lock to block concurrent updates at 1733397425369Preparing flush snapshotting stores in 1588230740 at 1733397425369Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733397425370 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733397425371 (+1 ms)Flushing 1588230740/info: creating writer at 1733397425371Flushing 1588230740/info: appending metadata at 1733397425389 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733397425389Flushing 1588230740/ns: creating writer at 1733397425795 (+406 ms)Flushing 1588230740/ns: appending metadata at 1733397425809 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733397425809Flushing 1588230740/rep_barrier: creating writer at 1733397426221 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733397426238 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733397426238Flushing 1588230740/table: creating writer at 1733397426652 (+414 ms)Flushing 1588230740/table: appending metadata at 1733397426670 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733397426670Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a57aed8: reopening flushed file at 1733397427082 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58eb93bd: reopening flushed file at 1733397427089 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b9ec985: reopening flushed file at 1733397427093 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@449656a4: reopening flushed file at 1733397427096 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1731ms, sequenceid=90, compaction requested=false at 1733397427101 (+5 ms)Writing region close event to WAL at 1733397427102 (+1 ms)Running coprocessor post-close hooks at 1733397427104 (+2 ms)Closed at 1733397427105 (+1 ms) 2024-12-05T11:17:07,105 DEBUG [RS_CLOSE_META-regionserver/6706a97ae65f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T11:17:07,172 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(976): stopping server 6706a97ae65f,42931,1733397243639; all regions closed. 2024-12-05T11:17:07,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741834_1010 (size=25344) 2024-12-05T11:17:07,184 DEBUG [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/oldWALs 2024-12-05T11:17:07,184 INFO [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6706a97ae65f%2C42931%2C1733397243639.meta:.meta(num 1733397247974) 2024-12-05T11:17:07,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741832_1008 (size=17901291) 2024-12-05T11:17:07,188 DEBUG [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/oldWALs 2024-12-05T11:17:07,188 INFO [RS:0;6706a97ae65f:42931 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6706a97ae65f%2C42931%2C1733397243639:(num 1733397246909) 2024-12-05T11:17:07,188 DEBUG [RS:0;6706a97ae65f:42931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T11:17:07,188 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T11:17:07,188 INFO [RS:0;6706a97ae65f:42931 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T11:17:07,188 INFO [RS:0;6706a97ae65f:42931 {}] hbase.ChoreService(370): Chore service for: regionserver/6706a97ae65f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T11:17:07,189 INFO [RS:0;6706a97ae65f:42931 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T11:17:07,189 INFO [regionserver/6706a97ae65f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T11:17:07,189 INFO [RS:0;6706a97ae65f:42931 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42931 2024-12-05T11:17:07,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6706a97ae65f,42931,1733397243639 2024-12-05T11:17:07,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T11:17:07,257 INFO [RS:0;6706a97ae65f:42931 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T11:17:07,268 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6706a97ae65f,42931,1733397243639] 2024-12-05T11:17:07,358 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6706a97ae65f,42931,1733397243639 already deleted, retry=false 2024-12-05T11:17:07,358 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6706a97ae65f,42931,1733397243639 expired; onlineServers=0 2024-12-05T11:17:07,358 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6706a97ae65f,38049,1733397242931' ***** 2024-12-05T11:17:07,358 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T11:17:07,359 INFO [M:0;6706a97ae65f:38049 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T11:17:07,359 INFO [M:0;6706a97ae65f:38049 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T11:17:07,359 DEBUG [M:0;6706a97ae65f:38049 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T11:17:07,359 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T11:17:07,359 DEBUG [M:0;6706a97ae65f:38049 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T11:17:07,359 DEBUG [master/6706a97ae65f:0:becomeActiveMaster-HFileCleaner.small.0-1733397246625 {}] cleaner.HFileCleaner(306): Exit Thread[master/6706a97ae65f:0:becomeActiveMaster-HFileCleaner.small.0-1733397246625,5,FailOnTimeoutGroup] 2024-12-05T11:17:07,359 DEBUG [master/6706a97ae65f:0:becomeActiveMaster-HFileCleaner.large.0-1733397246624 {}] cleaner.HFileCleaner(306): Exit Thread[master/6706a97ae65f:0:becomeActiveMaster-HFileCleaner.large.0-1733397246624,5,FailOnTimeoutGroup] 2024-12-05T11:17:07,359 INFO [M:0;6706a97ae65f:38049 {}] hbase.ChoreService(370): Chore service for: master/6706a97ae65f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T11:17:07,359 INFO [M:0;6706a97ae65f:38049 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T11:17:07,359 DEBUG [M:0;6706a97ae65f:38049 {}] master.HMaster(1795): Stopping service threads 2024-12-05T11:17:07,359 INFO [M:0;6706a97ae65f:38049 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T11:17:07,360 INFO [M:0;6706a97ae65f:38049 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T11:17:07,360 ERROR [M:0;6706a97ae65f:38049 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-05T11:17:07,360 INFO [M:0;6706a97ae65f:38049 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T11:17:07,360 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T11:17:07,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:17:07,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42931-0x101a532dcc70001, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:17:07,368 INFO [RS:0;6706a97ae65f:42931 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T11:17:07,369 INFO [RS:0;6706a97ae65f:42931 {}] regionserver.HRegionServer(1031): Exiting; stopping=6706a97ae65f,42931,1733397243639; zookeeper connection closed. 2024-12-05T11:17:07,369 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@76dcf8cc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@76dcf8cc 2024-12-05T11:17:07,369 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T11:17:07,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T11:17:07,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T11:17:07,383 DEBUG [M:0;6706a97ae65f:38049 {}] zookeeper.ZKUtil(347): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T11:17:07,383 WARN [M:0;6706a97ae65f:38049 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T11:17:07,384 INFO [M:0;6706a97ae65f:38049 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/.lastflushedseqids 2024-12-05T11:17:07,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742424_1600 (size=72) 2024-12-05T11:17:07,793 INFO [M:0;6706a97ae65f:38049 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T11:17:07,794 INFO [M:0;6706a97ae65f:38049 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T11:17:07,794 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T11:17:07,794 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:17:07,794 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:17:07,794 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T11:17:07,794 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:17:07,795 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=755.93 KB heapSize=924.62 KB 2024-12-05T11:17:07,819 DEBUG [M:0;6706a97ae65f:38049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c74bbeda26ce45de8121e125baaa8c5e is 82, key is hbase:meta,,1/info:regioninfo/1733397248047/Put/seqid=0 2024-12-05T11:17:07,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742425_1601 (size=5672) 2024-12-05T11:17:08,223 INFO [M:0;6706a97ae65f:38049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2087 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c74bbeda26ce45de8121e125baaa8c5e 2024-12-05T11:17:08,246 DEBUG [M:0;6706a97ae65f:38049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a0945e1aac54be4b17e0b142db8b800 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x85/proc:d/1733397394556/Put/seqid=0 2024-12-05T11:17:08,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742426_1602 (size=43259) 2024-12-05T11:17:08,650 INFO [M:0;6706a97ae65f:38049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=755.37 KB at sequenceid=2087 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a0945e1aac54be4b17e0b142db8b800 2024-12-05T11:17:08,657 INFO [M:0;6706a97ae65f:38049 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3a0945e1aac54be4b17e0b142db8b800 2024-12-05T11:17:08,680 DEBUG [M:0;6706a97ae65f:38049 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f51b7dad94146cdbc31f77fc7c7f25f is 69, key is 6706a97ae65f,42931,1733397243639/rs:state/1733397246681/Put/seqid=0 2024-12-05T11:17:08,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073742427_1603 (size=5156) 2024-12-05T11:17:09,086 INFO [M:0;6706a97ae65f:38049 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2087 (bloomFilter=true), to=hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f51b7dad94146cdbc31f77fc7c7f25f 2024-12-05T11:17:09,089 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c74bbeda26ce45de8121e125baaa8c5e as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c74bbeda26ce45de8121e125baaa8c5e 2024-12-05T11:17:09,092 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c74bbeda26ce45de8121e125baaa8c5e, entries=8, sequenceid=2087, filesize=5.5 K 2024-12-05T11:17:09,092 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a0945e1aac54be4b17e0b142db8b800 as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a0945e1aac54be4b17e0b142db8b800 2024-12-05T11:17:09,094 INFO [M:0;6706a97ae65f:38049 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3a0945e1aac54be4b17e0b142db8b800 2024-12-05T11:17:09,095 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a0945e1aac54be4b17e0b142db8b800, entries=158, sequenceid=2087, filesize=42.2 K 2024-12-05T11:17:09,095 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f51b7dad94146cdbc31f77fc7c7f25f as hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8f51b7dad94146cdbc31f77fc7c7f25f 2024-12-05T11:17:09,098 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35859/user/jenkins/test-data/cec2171e-ac59-3ce0-2e49-0e89d9aba6f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8f51b7dad94146cdbc31f77fc7c7f25f, entries=1, sequenceid=2087, filesize=5.0 K 2024-12-05T11:17:09,098 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(3140): Finished flush of dataSize ~755.93 KB/774068, heapSize ~924.32 KB/946504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1304ms, sequenceid=2087, compaction requested=false 2024-12-05T11:17:09,099 INFO [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T11:17:09,099 DEBUG [M:0;6706a97ae65f:38049 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733397427794Disabling compacts and flushes for region at 1733397427794Disabling writes for close at 1733397427794Obtaining lock to block concurrent updates at 1733397427795 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733397427795Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=774068, getHeapSize=946744, getOffHeapSize=0, getCellsCount=2270 at 1733397427795Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733397427797 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733397427797Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733397427819 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733397427819Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733397428227 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733397428245 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733397428245Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733397428657 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733397428680 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733397428680Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bc9dd31: reopening flushed file at 1733397429088 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1738dd61: reopening flushed file at 1733397429092 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b84bd52: reopening flushed file at 1733397429095 (+3 ms)Finished flush of dataSize ~755.93 KB/774068, heapSize ~924.32 KB/946504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1304ms, sequenceid=2087, compaction requested=false at 1733397429098 (+3 ms)Writing region close event to WAL at 1733397429099 (+1 ms)Closed at 1733397429099 2024-12-05T11:17:09,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42527 is added to blk_1073741830_1006 (size=910874) 2024-12-05T11:17:09,102 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T11:17:09,102 INFO [M:0;6706a97ae65f:38049 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T11:17:09,102 INFO [M:0;6706a97ae65f:38049 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38049 2024-12-05T11:17:09,103 INFO [M:0;6706a97ae65f:38049 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T11:17:09,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:17:09,209 INFO [M:0;6706a97ae65f:38049 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T11:17:09,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38049-0x101a532dcc70000, quorum=127.0.0.1:50124, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T11:17:09,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70061d7e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T11:17:09,222 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T11:17:09,222 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T11:17:09,222 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T11:17:09,222 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/hadoop.log.dir/,STOPPED} 2024-12-05T11:17:09,225 WARN [BP-320430523-172.17.0.2-1733397238870 heartbeating to localhost/127.0.0.1:35859 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T11:17:09,225 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T11:17:09,225 WARN [BP-320430523-172.17.0.2-1733397238870 heartbeating to localhost/127.0.0.1:35859 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-320430523-172.17.0.2-1733397238870 (Datanode Uuid bb5f54f6-9e96-4b7a-8f8a-b1bea9f6a6a2) service to localhost/127.0.0.1:35859 2024-12-05T11:17:09,225 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T11:17:09,227 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/data/data1/current/BP-320430523-172.17.0.2-1733397238870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T11:17:09,228 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/cluster_2f7e750b-a635-371a-25b5-66808acd3e61/data/data2/current/BP-320430523-172.17.0.2-1733397238870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T11:17:09,228 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T11:17:09,235 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f7463d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T11:17:09,236 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T11:17:09,236 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T11:17:09,236 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T11:17:09,236 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2656b75-e0c9-0472-b646-1c12044d366a/hadoop.log.dir/,STOPPED} 2024-12-05T11:17:09,252 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T11:17:09,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down